Skip to main content

common/parser_tools/
content_parser.rs

1use crate::entities::{Alignment, ListStyle, MarkerType, SemanticRole, TextDirection};
2use crate::parser_tools::djot_options::DjotImportOptions;
3
4/// An inline image recovered by a parser, before it becomes an `ImageAnchor`.
5///
6/// `src` is whatever the source document pointed at — a relative path, a bare
7/// name, a URL. Parsers do not resolve it; that is the embedding application's
8/// job, and this crate never touches the filesystem.
9#[derive(Debug, Clone, Default, PartialEq, Eq)]
10pub struct ParsedImage {
11    pub src: String,
12    pub alt: String,
13    /// Display size in pixels, `0` when the source did not state one.
14    ///
15    /// Djot and HTML both carry these as attributes (`{width=800}`,
16    /// `<img width=…>`); Markdown has no syntax for them, so a Markdown import
17    /// leaves both zero and the caller supplies intrinsic dimensions.
18    pub width: i64,
19    pub height: i64,
20}
21
22/// A parsed inline span with formatting info.
23///
24/// A span carries *either* text or an image, never both: an image span's `text`
25/// is empty and its description lives in [`ParsedImage::alt`], so alt text
26/// cannot leak into the block's prose (and therefore cannot be counted as
27/// manuscript words or matched by a search).
28#[derive(Debug, Clone, Default)]
29pub struct ParsedSpan {
30    pub text: String,
31    pub bold: bool,
32    pub italic: bool,
33    pub underline: bool,
34    pub strikeout: bool,
35    pub code: bool,
36    /// Superscript (djot `^x^`). Maps to `CharVerticalAlignment::SuperScript`.
37    pub superscript: bool,
38    /// Subscript (djot `~x~`). Maps to `CharVerticalAlignment::SubScript`.
39    pub subscript: bool,
40    pub link_href: Option<String>,
41    /// Set when this span is an inline image rather than text.
42    pub image: Option<ParsedImage>,
43    /// Set when this span is a footnote *reference* rather than text — the
44    /// label naming the note, never the number a reader sees.
45    pub footnote_ref: Option<String>,
46}
47
48/// A parsed table cell containing inline spans.
49#[derive(Debug, Clone)]
50pub struct ParsedTableCell {
51    pub spans: Vec<ParsedSpan>,
52}
53
54/// A parsed table extracted from markdown or HTML.
55#[derive(Debug, Clone)]
56pub struct ParsedTable {
57    /// Number of header rows (typically 1 for markdown tables).
58    pub header_rows: usize,
59    /// All rows (header + body), each containing cells with their inline spans.
60    pub rows: Vec<Vec<ParsedTableCell>>,
61    /// Blockquote nesting depth at the point the table appeared
62    /// (0 = not inside a blockquote), mirroring `ParsedBlock::blockquote_depth`.
63    pub blockquote_depth: u32,
64    /// How many of those quotations open at the table, mirroring
65    /// `ParsedBlock::blockquote_opens`.
66    pub blockquote_opens: u32,
67}
68
69impl ParsedTable {
70    /// How many columns the table needs to hold every cell it has: as many as its widest
71    /// row has cells. A row shorter than that is completed with empty cells by whoever
72    /// builds the table.
73    ///
74    /// The first row does not say it. The rows of a Djot table differ in length when their
75    /// source does, and an HTML table's first row is often one heading cell spanning every
76    /// column: a table sized from its first row kept the cells of a longer row in the model,
77    /// where they showed, while every export writes a row up to the table's width only, so
78    /// the first save dropped them.
79    pub fn column_count(&self) -> usize {
80        self.rows.iter().map(Vec::len).max().unwrap_or(0)
81    }
82}
83
84/// How many cells a table may reach once its short rows are completed (see
85/// [`ParsedTable::column_count`]), however few it has of its own: a table this size loads
86/// in a moment.
87const SQUARED_CELLS_ALWAYS: usize = 4096;
88
89/// How many times the cells it has of its own a larger table may reach once its short
90/// rows are completed.
91///
92/// A table whose rows differ in length because its source wrote them so, a wide heading
93/// row over rows of one cell or a few, grows to about as many times its own cells as it has
94/// columns: a dozen or so for a table a writer makes. The tables that made hundreds of
95/// thousands of cells out of a few kilobytes grow hundreds of times. At four times, a table
96/// of ten columns heading five hundred rows of one cell was read as paragraphs, its grid
97/// lost for a table that loads in a few milliseconds.
98const SQUARED_CELLS_PER_OWN_CELL: usize = 16;
99
100/// Push `table`, which has `own_cells` cells of its own in its source, onto `elements`: as
101/// a table, or as the paragraphs of its cells when completing its short rows would make it
102/// many times that.
103///
104/// Every row of a loaded table is completed to the widest one. A few kilobytes of a table
105/// whose first row is wide and whose other rows hold one cell each, or of HTML cells each
106/// spanning a thousand columns, made hundreds of thousands of empty cells, and held a load
107/// or a paste for seconds. Such a table is read as the paragraphs of its cells that hold
108/// anything, in reading order, at the table's quotation depth: every word stays, the grid
109/// goes.
110fn push_table(elements: &mut Vec<ParsedElement>, table: ParsedTable, own_cells: usize) {
111    let squared = table.rows.len().saturating_mul(table.column_count());
112    let allowed = SQUARED_CELLS_ALWAYS.max(own_cells.saturating_mul(SQUARED_CELLS_PER_OWN_CELL));
113    if squared <= allowed {
114        elements.push(ParsedElement::Table(table));
115        return;
116    }
117    elements.extend(
118        ParsedElement::flatten_to_blocks(vec![ParsedElement::Table(table)])
119            .into_iter()
120            .filter(|block| spans_carry_content(&block.spans))
121            .map(ParsedElement::Block),
122    );
123}
124
125/// Record that a quotation opened at the first block or table of `elements` from `first`
126/// on: the quotation that began where `elements` stood at `first` and closes now. Nothing
127/// when the quotation holds none.
128fn mark_quotation_opened(elements: &mut [ParsedElement], first: usize) {
129    let Some(rest) = elements.get_mut(first..) else {
130        return;
131    };
132    for element in rest {
133        match element {
134            ParsedElement::Block(block) => {
135                block.blockquote_opens += 1;
136                return;
137            }
138            ParsedElement::Table(table) => {
139                table.blockquote_opens += 1;
140                return;
141            }
142            ParsedElement::FootnoteDefinition { .. } => {}
143        }
144    }
145}
146
147/// The cells a table's rows hold, for [`push_table`].
148fn cells_in_rows(rows: &[Vec<ParsedTableCell>]) -> usize {
149    rows.iter().map(Vec::len).sum()
150}
151
152/// A parsed element: either a block or a table.
153#[derive(Debug, Clone)]
154pub enum ParsedElement {
155    Block(ParsedBlock),
156    Table(ParsedTable),
157    /// A footnote definition: the label its references name, and the blocks
158    /// making up its body.
159    ///
160    /// Separate from `Block` because a definition is not part of the flow it was
161    /// written in — it belongs wherever the output format puts notes, which the
162    /// importer expresses by giving it a detached frame of its own.
163    FootnoteDefinition {
164        label: String,
165        blocks: Vec<ParsedBlock>,
166    },
167}
168
169impl ParsedElement {
170    /// Extract blocks, flattening tables into one block per cell.
171    /// Use when table structure is not needed.
172    pub fn flatten_to_blocks(elements: Vec<ParsedElement>) -> Vec<ParsedBlock> {
173        let mut blocks = Vec::new();
174        for elem in elements {
175            match elem {
176                ParsedElement::Block(b) => blocks.push(b),
177                // A definition is not part of the flow it was written in — it
178                // belongs wherever the format puts notes. Flattening it in
179                // would splice a note's body into the prose at the point the
180                // definition happened to be typed.
181                ParsedElement::FootnoteDefinition { .. } => {}
182                ParsedElement::Table(t) => {
183                    for row in t.rows {
184                        for cell in row {
185                            blocks.push(ParsedBlock {
186                                spans: cell.spans,
187                                heading_level: None,
188                                list_style: None,
189                                list_indent: 0,
190                                list_prefix: String::new(),
191                                list_suffix: String::new(),
192                                list_start: None,
193                                marker: None,
194                                is_code_block: false,
195                                code_language: None,
196                                blockquote_depth: t.blockquote_depth,
197                                blockquote_opens: 0,
198                                line_height: None,
199                                non_breakable_lines: None,
200                                page_break_before: None,
201                                front_matter: None,
202                                direction: None,
203                                background_color: None,
204                                alignment: None,
205                                top_margin: None,
206                                text_indent: None,
207                                semantic_role: None,
208                            });
209                        }
210                    }
211                }
212            }
213        }
214        if blocks.is_empty() {
215            blocks.push(ParsedBlock {
216                spans: vec![ParsedSpan {
217                    text: String::new(),
218                    ..Default::default()
219                }],
220                heading_level: None,
221                list_style: None,
222                list_indent: 0,
223                list_prefix: String::new(),
224                list_suffix: String::new(),
225                list_start: None,
226                marker: None,
227                is_code_block: false,
228                code_language: None,
229                blockquote_depth: 0,
230                blockquote_opens: 0,
231                line_height: None,
232                non_breakable_lines: None,
233                page_break_before: None,
234                front_matter: None,
235                direction: None,
236                background_color: None,
237                alignment: None,
238                top_margin: None,
239                text_indent: None,
240                semantic_role: None,
241            });
242        }
243        blocks
244    }
245}
246
247/// A parsed block (paragraph, heading, list item, code block)
248///
249/// `Default` is an unformatted, empty paragraph — the shape a caller wants when
250/// it has prose and no structure to give it. Deriving it also means a field
251/// added later reaches those callers as "absent" rather than as a compile error
252/// they would fix by guessing a value.
253#[derive(Debug, Clone, Default)]
254pub struct ParsedBlock {
255    pub spans: Vec<ParsedSpan>,
256    pub heading_level: Option<i64>,
257    pub list_style: Option<ListStyle>,
258    pub list_indent: u32,
259    /// Ordered-list delimiter prefix (e.g. `"("` for djot `(1)` lists; empty
260    /// otherwise). Stored on the `List` entity for round-trip fidelity.
261    pub list_prefix: String,
262    /// Ordered-list delimiter suffix (`"."` for `1.`, `")"` for `1)`/`(1)`;
263    /// empty for unordered lists).
264    pub list_suffix: String,
265    /// The number the first item of the list this item belongs to wears, when it is not 1
266    /// (Djot and Markdown `3.`, HTML `<ol start="3">`). Every item of a list carries it.
267    /// Stored on the `List` entity.
268    pub list_start: Option<i64>,
269    /// Task-list checkbox marker (djot `- [ ]` / `- [x]`). Maps to
270    /// `Block.fmt_marker`. `None` for non-task blocks.
271    pub marker: Option<MarkerType>,
272    pub is_code_block: bool,
273    pub code_language: Option<String>,
274    pub blockquote_depth: u32,
275    /// How many of the quotations the block stands in open at it, the innermost ones:
276    /// quotations as deep as the element's before it but not the same ones. `0` when the
277    /// block goes on in the quotations the element before it stands in, as deep as it
278    /// stands. Two quotations one after the other (Djot and Markdown `> a` and `> b` with a
279    /// blank line between, HTML `<blockquote>` twice) were read as one, since the depth
280    /// alone does not tell them apart.
281    pub blockquote_opens: u32,
282    pub line_height: Option<i64>,
283    pub non_breakable_lines: Option<bool>,
284    /// Start this block on a new page (djot `{page_break_before=true}`). Maps to
285    /// `Block.fmt_page_break_before`. `None` when absent.
286    pub page_break_before: Option<bool>,
287    /// This block opens the document as front matter (djot `{front_matter=true}`). Maps to
288    /// `Block.fmt_front_matter`. `None` when absent.
289    pub front_matter: Option<bool>,
290    pub direction: Option<TextDirection>,
291    pub background_color: Option<String>,
292    /// Paragraph alignment (djot `{alignment=left|right|center|justify}`). Maps
293    /// to `Block.fmt_alignment`. `None` when no alignment attribute is present.
294    pub alignment: Option<Alignment>,
295    /// This block's own space-above (djot `{top_margin=<int>}`). Maps to
296    /// `Block.fmt_top_margin` and overrides the document-wide paragraph
297    /// spacing for this block alone. `None` when absent.
298    pub top_margin: Option<i64>,
299    /// This block's own first-line indent (djot `{text_indent=<int>}`). Maps to
300    /// `Block.fmt_text_indent` and overrides the document-wide first-line
301    /// indent for this block alone. `None` when absent.
302    pub text_indent: Option<i64>,
303    /// The enclosing blockquote's semantic role (djot `{semantic_role=epigraph}`),
304    /// written on the quote's first block because block attributes are the only channel
305    /// djot offers. The importer lifts it onto the `Frame`, where it belongs.
306    pub semantic_role: Option<SemanticRole>,
307}
308
309impl ParsedBlock {
310    /// Returns `true` when this block carries no block-level formatting,
311    /// meaning its content is purely inline.
312    pub fn is_inline_only(&self) -> bool {
313        self.heading_level.is_none()
314            && self.list_style.is_none()
315            && !self.is_code_block
316            && self.blockquote_depth == 0
317            && self.line_height.is_none()
318            && self.non_breakable_lines.is_none()
319            && self.page_break_before.is_none()
320            && self.front_matter.is_none()
321            && self.direction.is_none()
322            && self.background_color.is_none()
323            && self.alignment.is_none()
324            && self.top_margin.is_none()
325            && self.text_indent.is_none()
326    }
327}
328
329// ─── Markdown parsing ────────────────────────────────────────────────
330
331/// Labels referenced as `[^label]` in `markdown` with no `[^label]: ...`
332/// definition anywhere in the document.
333///
334/// pulldown-cmark's default footnote mode (`Options::ENABLE_FOOTNOTES`, i.e.
335/// GitHub's syntax) only emits `Event::FootnoteReference` for a label some
336/// `Tag::FootnoteDefinition` actually defines (verified against 0.13) — an
337/// undefined reference silently decomposes into three ordinary `Text` events
338/// (`"["`, `"^label"`, `"]"`) instead, indistinguishable from a reader having
339/// typed literal brackets. `Options::ENABLE_OLD_FOOTNOTES` recognises it, but
340/// as `parse_markdown`'s doc comment explains, trades away multi-paragraph
341/// definition bodies to get there. Neither flag alone gives both, so
342/// `parse_markdown` hands pulldown-cmark a synthetic empty definition for
343/// every such label instead — real enough for it to recognise the reference,
344/// thrown away again before `elements` is returned.
345///
346/// Finding "every such label" is two parts: which labels pulldown-cmark's own
347/// block scanner (a first pass, at the caller's `options`) already recognises
348/// as truly defined, and which `[^…]`-shaped spans exist in the raw text at
349/// all. The second part is a plain scan, not a parser — it can over-match
350/// (inside a code span, inside a fenced block, a coincidental `[^x]:` in the
351/// middle of a sentence that real block parsing would never treat as a
352/// definition), but over-matching only ever produces an unused synthetic
353/// definition: pulldown-cmark's real, correct parse of the augmented text is
354/// what decides whether any span actually becomes a reference, and
355/// `parse_markdown` drops every synthesized definition regardless of whether
356/// that happened.
357fn dangling_footnote_labels(
358    markdown: &str,
359    options: pulldown_cmark::Options,
360) -> std::collections::BTreeSet<String> {
361    use pulldown_cmark::{Event, Parser, Tag};
362
363    let mut defined: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
364    for event in Parser::new_ext(markdown, options) {
365        if let Event::Start(Tag::FootnoteDefinition(label)) = event {
366            defined.insert(label.to_string());
367        }
368    }
369
370    let mut referenced: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
371    let bytes = markdown.as_bytes();
372    let mut search_from = 0usize;
373    while let Some(rel) = markdown[search_from..].find("[^") {
374        let open = search_from + rel;
375        let label_start = open + 2;
376        let Some(close_rel) = markdown[label_start..].find(']') else {
377            break;
378        };
379        let close = label_start + close_rel;
380        let label = &markdown[label_start..close];
381        // `]:` immediately after is definition syntax, not a reference —
382        // pulldown-cmark's own scan above already supplied the ground truth
383        // for which labels are truly defined; this only needs to avoid
384        // counting a definition's own label as a "reference" candidate.
385        let looks_like_definition = bytes.get(close + 1) == Some(&b':');
386        if !label.is_empty() && !looks_like_definition && !label.chars().any(char::is_whitespace) {
387            referenced.insert(label.to_string());
388        }
389        search_from = close + 1;
390    }
391
392    referenced.difference(&defined).cloned().collect()
393}
394
395/// Read `markdown` into blocks and tables.
396///
397/// The parser reads nested containers without recursing, whatever their depth. The
398/// quotations the text holds are kept within
399/// [`MAX_QUOTE_LEVELS`](crate::parser_tools::quote_depth::MAX_QUOTE_LEVELS) (see
400/// `quote_depth`): the writers walk one call deeper for each, and a text nested 500
401/// quotations deep loaded, then aborted the process at its first save.
402pub fn parse_markdown(markdown: &str) -> Vec<ParsedElement> {
403    use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
404
405    // ENABLE_FOOTNOTES is required for pulldown-cmark to parse `[^label]`
406    // references and `[^label]: body` definitions at all — without it both
407    // fall through as plain link-reference-shaped text. Deliberately NOT
408    // `ENABLE_OLD_FOOTNOTES` too: that variant makes a reference with no
409    // definition survive (see `dangling_footnote_labels`), but in exchange
410    // breaks multi-paragraph definition bodies — a blank line before an
411    // indented continuation escapes the definition and becomes a sibling
412    // code block, which is exactly the shape this crate's own Markdown
413    // exporter writes for a note with more than one paragraph. Getting both
414    // is `dangling_footnote_labels`'s job, not an `Options` flag's.
415    let options = Options::ENABLE_STRIKETHROUGH
416        | Options::ENABLE_TABLES
417        | Options::ENABLE_TASKLISTS
418        | Options::ENABLE_FOOTNOTES;
419
420    // See `dangling_footnote_labels`: pulldown-cmark's GFM-style footnotes
421    // only recognise `[^label]` as a reference when some definition exists
422    // for it anywhere in the document, so an undefined one — the normal
423    // state for a host that owns note bodies itself — has to be given a
424    // throwaway definition before parsing, or it silently becomes the
425    // literal text "[^label]" instead of surviving as a reference.
426    let dangling = dangling_footnote_labels(markdown, options);
427    let augmented_owner;
428    let source: &str = if dangling.is_empty() {
429        markdown
430    } else {
431        augmented_owner = dangling
432            .iter()
433            .fold(markdown.to_string(), |mut acc, label| {
434                acc.push_str("\n\n[^");
435                acc.push_str(label);
436                acc.push_str("]:\n");
437                acc
438            });
439        &augmented_owner
440    };
441    let parser = Parser::new_ext(source, options);
442
443    let mut elements: Vec<ParsedElement> = Vec::new();
444    let mut current_spans: Vec<ParsedSpan> = Vec::new();
445    let mut current_heading: Option<i64> = None;
446    let mut current_list_style: Option<ListStyle> = None;
447    let mut current_list_start: Option<i64> = None;
448    let mut is_code_block = false;
449    let mut code_language: Option<String> = None;
450    let mut blockquote_depth: u32 = 0;
451    // Where `elements` stood as each open quotation began (see `mark_quotation_opened`).
452    let mut quotations_open: Vec<usize> = Vec::new();
453    let mut in_block = false;
454
455    // Formatting state stack
456    let mut bold = false;
457    let mut italic = false;
458    let mut strikeout = false;
459    let mut link_href: Option<String> = None;
460    // Set between an image's Start and End; its alt text arrives as Text events.
461    let mut pending_image: Option<ParsedImage> = None;
462
463    // The label and element index of the footnote definition currently open,
464    // if any. Mirrors `parse_djot`'s `footnote_open`: a definition's body is
465    // ordinary block content, parsed by the same machinery as everything
466    // else, then lifted back out at `End` into its own top-level element so
467    // it never becomes part of the flow it was written in. CommonMark
468    // footnote definitions cannot nest, so one slot suffices.
469    let mut footnote_open: Option<(String, usize)> = None;
470
471    // List style stack for nested lists (also tracks nesting depth), and the start of each
472    // of those lists beside it.
473    let mut list_stack: Vec<Option<ListStyle>> = Vec::new();
474    let mut list_start_stack: Vec<Option<i64>> = Vec::new();
475    let mut current_list_indent: u32 = 0;
476
477    // Table tracking state
478    let mut in_table = false;
479    let mut in_table_head = false;
480    let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
481    let mut current_row_cells: Vec<ParsedTableCell> = Vec::new();
482    let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
483    let mut table_header_rows: usize = 0;
484
485    for event in parser {
486        match event {
487            Event::Start(Tag::Paragraph) => {
488                in_block = true;
489                current_heading = None;
490                is_code_block = false;
491            }
492            Event::End(TagEnd::Paragraph) => {
493                if !current_spans.is_empty() || in_block {
494                    elements.push(ParsedElement::Block(ParsedBlock {
495                        spans: std::mem::take(&mut current_spans),
496                        heading_level: current_heading.take(),
497                        list_style: current_list_style.clone(),
498                        list_indent: current_list_indent,
499                        list_prefix: String::new(),
500                        list_suffix: String::new(),
501                        list_start: current_list_start,
502                        marker: None,
503                        is_code_block: false,
504                        code_language: None,
505                        blockquote_depth,
506                        blockquote_opens: 0,
507                        line_height: None,
508                        non_breakable_lines: None,
509                        page_break_before: None,
510                        front_matter: None,
511                        direction: None,
512                        background_color: None,
513                        alignment: None,
514                        top_margin: None,
515                        text_indent: None,
516                        semantic_role: None,
517                    }));
518                }
519                in_block = false;
520                current_list_style = None;
521                current_list_start = None;
522            }
523            Event::Start(Tag::Heading { level, .. }) => {
524                in_block = true;
525                current_heading = Some(heading_level_to_i64(level));
526                is_code_block = false;
527            }
528            Event::End(TagEnd::Heading(_)) => {
529                elements.push(ParsedElement::Block(ParsedBlock {
530                    spans: std::mem::take(&mut current_spans),
531                    heading_level: current_heading.take(),
532                    list_style: None,
533                    list_indent: 0,
534                    list_prefix: String::new(),
535                    list_suffix: String::new(),
536                    list_start: None,
537                    marker: None,
538                    is_code_block: false,
539                    code_language: None,
540                    blockquote_depth,
541                    blockquote_opens: 0,
542                    line_height: None,
543                    non_breakable_lines: None,
544                    page_break_before: None,
545                    front_matter: None,
546                    direction: None,
547                    background_color: None,
548                    alignment: None,
549                    top_margin: None,
550                    text_indent: None,
551                    semantic_role: None,
552                }));
553                in_block = false;
554            }
555            Event::Start(Tag::List(ordered)) => {
556                let style = if ordered.is_some() {
557                    Some(ListStyle::Decimal)
558                } else {
559                    Some(ListStyle::Disc)
560                };
561                list_stack.push(style);
562                list_start_stack.push(ordered.and_then(list_start_of));
563            }
564            Event::End(TagEnd::List(_)) => {
565                list_stack.pop();
566                list_start_stack.pop();
567            }
568            Event::Start(Tag::Item) => {
569                // Flush any accumulated spans from the parent item before
570                // starting a child item in a tight list
571                if !current_spans.is_empty() {
572                    elements.push(ParsedElement::Block(ParsedBlock {
573                        spans: std::mem::take(&mut current_spans),
574                        heading_level: None,
575                        list_style: current_list_style.clone(),
576                        list_indent: current_list_indent,
577                        list_prefix: String::new(),
578                        list_suffix: String::new(),
579                        list_start: current_list_start,
580                        marker: None,
581                        is_code_block: false,
582                        code_language: None,
583                        blockquote_depth,
584                        blockquote_opens: 0,
585                        line_height: None,
586                        non_breakable_lines: None,
587                        page_break_before: None,
588                        front_matter: None,
589                        direction: None,
590                        background_color: None,
591                        alignment: None,
592                        top_margin: None,
593                        text_indent: None,
594                        semantic_role: None,
595                    }));
596                }
597                in_block = true;
598                current_list_style = list_stack.last().cloned().flatten();
599                current_list_start = list_start_stack.last().copied().flatten();
600                current_list_indent = if list_stack.is_empty() {
601                    0
602                } else {
603                    (list_stack.len() - 1) as u32
604                };
605            }
606            Event::End(TagEnd::Item) => {
607                // The paragraph inside the item will have already been flushed,
608                // but if there was no inner paragraph (tight list), flush now.
609                if !current_spans.is_empty() {
610                    elements.push(ParsedElement::Block(ParsedBlock {
611                        spans: std::mem::take(&mut current_spans),
612                        heading_level: None,
613                        list_style: current_list_style.clone(),
614                        list_indent: current_list_indent,
615                        list_prefix: String::new(),
616                        list_suffix: String::new(),
617                        list_start: current_list_start,
618                        marker: None,
619                        is_code_block: false,
620                        code_language: None,
621                        blockquote_depth,
622                        blockquote_opens: 0,
623                        line_height: None,
624                        non_breakable_lines: None,
625                        page_break_before: None,
626                        front_matter: None,
627                        direction: None,
628                        background_color: None,
629                        alignment: None,
630                        top_margin: None,
631                        text_indent: None,
632                        semantic_role: None,
633                    }));
634                }
635                in_block = false;
636                current_list_style = None;
637                current_list_start = None;
638            }
639            Event::Start(Tag::CodeBlock(kind)) => {
640                in_block = true;
641                is_code_block = true;
642                code_language = match &kind {
643                    pulldown_cmark::CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
644                        Some(lang.to_string())
645                    }
646                    _ => None,
647                };
648            }
649            Event::End(TagEnd::CodeBlock) => {
650                // pulldown-cmark appends a trailing '\n' to code block text — strip it
651                if let Some(last) = current_spans.last_mut()
652                    && last.text.ends_with('\n')
653                {
654                    last.text.truncate(last.text.len() - 1);
655                }
656                elements.push(ParsedElement::Block(ParsedBlock {
657                    spans: std::mem::take(&mut current_spans),
658                    heading_level: None,
659                    list_style: None,
660                    list_indent: 0,
661                    list_prefix: String::new(),
662                    list_suffix: String::new(),
663                    list_start: None,
664                    marker: None,
665                    is_code_block: true,
666                    code_language: code_language.take(),
667                    blockquote_depth,
668                    blockquote_opens: 0,
669                    line_height: None,
670                    non_breakable_lines: None,
671                    page_break_before: None,
672                    front_matter: None,
673                    direction: None,
674                    background_color: None,
675                    alignment: None,
676                    top_margin: None,
677                    text_indent: None,
678                    semantic_role: None,
679                }));
680                in_block = false;
681                is_code_block = false;
682            }
683            // ─── Table events ───────────────────────────────────────
684            Event::Start(Tag::Table(_)) => {
685                in_table = true;
686                in_table_head = false;
687                table_rows.clear();
688                current_row_cells.clear();
689                current_cell_spans.clear();
690                table_header_rows = 0;
691            }
692            Event::End(TagEnd::Table) => {
693                let own_cells = cells_in_rows(&table_rows);
694                push_table(
695                    &mut elements,
696                    ParsedTable {
697                        header_rows: table_header_rows,
698                        rows: std::mem::take(&mut table_rows),
699                        blockquote_depth,
700                        blockquote_opens: 0,
701                    },
702                    own_cells,
703                );
704                in_table = false;
705            }
706            Event::Start(Tag::TableHead) => {
707                in_table_head = true;
708                current_row_cells.clear();
709            }
710            Event::End(TagEnd::TableHead) => {
711                // Flush the header row
712                table_rows.push(std::mem::take(&mut current_row_cells));
713                table_header_rows += 1;
714                in_table_head = false;
715            }
716            Event::Start(Tag::TableRow) => {
717                current_row_cells.clear();
718            }
719            Event::End(TagEnd::TableRow) if !in_table_head => {
720                // Body rows only — header row is flushed in End(TableHead)
721                table_rows.push(std::mem::take(&mut current_row_cells));
722            }
723            Event::Start(Tag::TableCell) => {
724                current_cell_spans.clear();
725            }
726            Event::End(TagEnd::TableCell) => {
727                current_row_cells.push(ParsedTableCell {
728                    spans: std::mem::take(&mut current_cell_spans),
729                });
730            }
731            // ─── Inline formatting ──────────────────────────────────
732            Event::Start(Tag::Emphasis) => {
733                italic = true;
734            }
735            Event::End(TagEnd::Emphasis) => {
736                italic = false;
737            }
738            Event::Start(Tag::Strong) => {
739                bold = true;
740            }
741            Event::End(TagEnd::Strong) => {
742                bold = false;
743            }
744            Event::Start(Tag::Strikethrough) => {
745                strikeout = true;
746            }
747            Event::End(TagEnd::Strikethrough) => {
748                strikeout = false;
749            }
750            Event::Start(Tag::Link { dest_url, .. }) => {
751                link_href = Some(dest_url.to_string());
752            }
753            Event::End(TagEnd::Link) => {
754                link_href = None;
755            }
756            // Markdown has no syntax for display size, so width/height stay 0
757            // and the caller supplies the image's intrinsic dimensions.
758            Event::Start(Tag::Image { dest_url, .. }) => {
759                pending_image = Some(ParsedImage {
760                    src: dest_url.to_string(),
761                    alt: String::new(),
762                    width: 0,
763                    height: 0,
764                });
765            }
766            Event::End(TagEnd::Image) => {
767                if let Some(image) = pending_image.take() {
768                    let span = ParsedSpan {
769                        text: String::new(),
770                        bold,
771                        italic,
772                        underline: false,
773                        strikeout,
774                        code: false,
775                        superscript: false,
776                        subscript: false,
777                        link_href: link_href.clone(),
778                        image: Some(image),
779                        footnote_ref: None,
780                    };
781                    if in_table {
782                        current_cell_spans.push(span);
783                    } else {
784                        if !in_block {
785                            in_block = true;
786                        }
787                        current_spans.push(span);
788                    }
789                }
790            }
791            Event::Text(text) => {
792                // Inside an image this is its alt text, which pulldown-cmark
793                // emits as an ordinary Text event. Without this guard it fell
794                // through and landed in the paragraph as prose — the image was
795                // dropped and its description silently became manuscript text.
796                if let Some(img) = pending_image.as_mut() {
797                    img.alt.push_str(&text);
798                    continue;
799                }
800                let span = ParsedSpan {
801                    text: text.to_string(),
802                    bold,
803                    italic,
804                    underline: false,
805                    strikeout,
806                    code: is_code_block,
807                    superscript: false,
808                    subscript: false,
809                    link_href: link_href.clone(),
810                    image: None,
811                    footnote_ref: None,
812                };
813                if in_table {
814                    current_cell_spans.push(span);
815                } else {
816                    if !in_block {
817                        in_block = true;
818                    }
819                    current_spans.push(span);
820                }
821            }
822            Event::Code(text) => {
823                let span = ParsedSpan {
824                    text: text.to_string(),
825                    bold,
826                    italic,
827                    underline: false,
828                    strikeout,
829                    code: true,
830                    superscript: false,
831                    subscript: false,
832                    link_href: link_href.clone(),
833                    image: None,
834                    footnote_ref: None,
835                };
836                if in_table {
837                    current_cell_spans.push(span);
838                } else {
839                    if !in_block {
840                        in_block = true;
841                    }
842                    current_spans.push(span);
843                }
844            }
845            Event::SoftBreak => {
846                let span = ParsedSpan {
847                    text: " ".to_string(),
848                    bold,
849                    italic,
850                    underline: false,
851                    strikeout,
852                    code: false,
853                    superscript: false,
854                    subscript: false,
855                    link_href: link_href.clone(),
856                    image: None,
857                    footnote_ref: None,
858                };
859                if in_table {
860                    current_cell_spans.push(span);
861                } else {
862                    current_spans.push(span);
863                }
864            }
865            Event::HardBreak if !current_spans.is_empty() || in_block => {
866                // Finalize current block
867                elements.push(ParsedElement::Block(ParsedBlock {
868                    spans: std::mem::take(&mut current_spans),
869                    heading_level: current_heading.take(),
870                    list_style: current_list_style.clone(),
871                    list_indent: current_list_indent,
872                    list_prefix: String::new(),
873                    list_suffix: String::new(),
874                    list_start: current_list_start,
875                    marker: None,
876                    is_code_block,
877                    code_language: code_language.clone(),
878                    blockquote_depth,
879                    blockquote_opens: 0,
880                    line_height: None,
881                    non_breakable_lines: None,
882                    page_break_before: None,
883                    front_matter: None,
884                    direction: None,
885                    background_color: None,
886                    alignment: None,
887                    top_margin: None,
888                    text_indent: None,
889                    semantic_role: None,
890                }));
891            }
892            Event::Start(Tag::BlockQuote(_)) => {
893                blockquote_depth += 1;
894                quotations_open.push(elements.len());
895            }
896            Event::End(TagEnd::BlockQuote(_)) => {
897                blockquote_depth = blockquote_depth.saturating_sub(1);
898                if let Some(first) = quotations_open.pop() {
899                    mark_quotation_opened(&mut elements, first);
900                }
901            }
902            // ── Footnote definitions ──
903            //
904            // `[^label]: body` opens a block-level container; its body is
905            // ordinary paragraphs/lists/etc., pushed onto `elements` by the
906            // ordinary machinery above. `End` drains everything pushed since
907            // `Start` back out into one `FootnoteDefinition`, exactly as
908            // `parse_djot` does for the same `[^label]: body` syntax.
909            // Flushing any spans left open by an interrupted block first
910            // matches every other container boundary in this parser
911            // (List/Item/BlockQuote).
912            Event::Start(Tag::FootnoteDefinition(label)) => {
913                if !current_spans.is_empty() {
914                    elements.push(ParsedElement::Block(ParsedBlock {
915                        spans: std::mem::take(&mut current_spans),
916                        heading_level: current_heading.take(),
917                        list_style: current_list_style.clone(),
918                        list_indent: current_list_indent,
919                        list_prefix: String::new(),
920                        list_suffix: String::new(),
921                        list_start: current_list_start,
922                        marker: None,
923                        is_code_block: false,
924                        code_language: None,
925                        blockquote_depth,
926                        blockquote_opens: 0,
927                        line_height: None,
928                        non_breakable_lines: None,
929                        page_break_before: None,
930                        front_matter: None,
931                        direction: None,
932                        background_color: None,
933                        alignment: None,
934                        top_margin: None,
935                        text_indent: None,
936                        semantic_role: None,
937                    }));
938                }
939                footnote_open = Some((label.to_string(), elements.len()));
940            }
941            Event::End(TagEnd::FootnoteDefinition) => {
942                if !current_spans.is_empty() {
943                    elements.push(ParsedElement::Block(ParsedBlock {
944                        spans: std::mem::take(&mut current_spans),
945                        heading_level: current_heading.take(),
946                        list_style: current_list_style.clone(),
947                        list_indent: current_list_indent,
948                        list_prefix: String::new(),
949                        list_suffix: String::new(),
950                        list_start: current_list_start,
951                        marker: None,
952                        is_code_block: false,
953                        code_language: None,
954                        blockquote_depth,
955                        blockquote_opens: 0,
956                        line_height: None,
957                        non_breakable_lines: None,
958                        page_break_before: None,
959                        front_matter: None,
960                        direction: None,
961                        background_color: None,
962                        alignment: None,
963                        top_margin: None,
964                        text_indent: None,
965                        semantic_role: None,
966                    }));
967                }
968                if let Some((label, start)) = footnote_open.take() {
969                    let blocks: Vec<ParsedBlock> = elements
970                        .drain(start..)
971                        .filter_map(|e| match e {
972                            ParsedElement::Block(b) => Some(b),
973                            // A table inside a footnote is not representable
974                            // as note content, same limitation as djot.
975                            _ => None,
976                        })
977                        .collect();
978                    elements.push(ParsedElement::FootnoteDefinition { label, blocks });
979                }
980            }
981            // A footnote reference. pulldown-cmark emits this whether or not
982            // a matching `[^label]:` definition exists anywhere in the
983            // document — a dangling reference (the normal state for a host
984            // that owns note bodies itself) must survive just the same,
985            // mirroring `parse_djot`'s `E::FootnoteReference` handling.
986            Event::FootnoteReference(label) => {
987                let span = ParsedSpan {
988                    text: String::new(),
989                    bold,
990                    italic,
991                    underline: false,
992                    strikeout,
993                    code: false,
994                    superscript: false,
995                    subscript: false,
996                    link_href: link_href.clone(),
997                    image: None,
998                    footnote_ref: Some(label.to_string()),
999                };
1000                if in_table {
1001                    current_cell_spans.push(span);
1002                } else {
1003                    if !in_block {
1004                        in_block = true;
1005                    }
1006                    current_spans.push(span);
1007                }
1008            }
1009            _ => {}
1010        }
1011    }
1012
1013    // Flush any remaining content
1014    if !current_spans.is_empty() {
1015        elements.push(ParsedElement::Block(ParsedBlock {
1016            spans: std::mem::take(&mut current_spans),
1017            heading_level: current_heading,
1018            list_style: current_list_style,
1019            list_indent: current_list_indent,
1020            list_prefix: String::new(),
1021            list_suffix: String::new(),
1022            list_start: current_list_start,
1023            marker: None,
1024            is_code_block,
1025            code_language: code_language.take(),
1026            blockquote_depth,
1027            blockquote_opens: 0,
1028            line_height: None,
1029            non_breakable_lines: None,
1030            page_break_before: None,
1031            front_matter: None,
1032            direction: None,
1033            background_color: None,
1034            alignment: None,
1035            top_margin: None,
1036            text_indent: None,
1037            semantic_role: None,
1038        }));
1039    }
1040
1041    // Drop the throwaway definitions `dangling_footnote_labels` asked for —
1042    // they exist only to make pulldown-cmark recognise the reference as
1043    // real, never to become a note the document owns. `elements` cannot
1044    // become empty from this: a label reached `dangling` only via a
1045    // reference actually present in `markdown`, so its surrounding block
1046    // survives even with the synthetic definition gone.
1047    if !dangling.is_empty() {
1048        elements.retain(
1049            |e| !matches!(e, ParsedElement::FootnoteDefinition { label, .. } if dangling.contains(label)),
1050        );
1051    }
1052
1053    // If no elements were parsed, create a single empty paragraph
1054    if elements.is_empty() {
1055        elements.push(ParsedElement::Block(ParsedBlock {
1056            spans: vec![ParsedSpan {
1057                text: String::new(),
1058                ..Default::default()
1059            }],
1060            heading_level: None,
1061            list_style: None,
1062            list_indent: 0,
1063            list_prefix: String::new(),
1064            list_suffix: String::new(),
1065            list_start: None,
1066            marker: None,
1067            is_code_block: false,
1068            code_language: None,
1069            blockquote_depth: 0,
1070            blockquote_opens: 0,
1071            line_height: None,
1072            non_breakable_lines: None,
1073            page_break_before: None,
1074            front_matter: None,
1075            direction: None,
1076            background_color: None,
1077            alignment: None,
1078            top_margin: None,
1079            text_indent: None,
1080            semantic_role: None,
1081        }));
1082    }
1083
1084    // Quotations past the ceiling a document holds are flattened into the deepest kept
1085    // (see `quote_depth`): the writers walk one call deeper for each.
1086    crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1087    elements
1088}
1089
1090fn heading_level_to_i64(level: pulldown_cmark::HeadingLevel) -> i64 {
1091    use pulldown_cmark::HeadingLevel;
1092    match level {
1093        HeadingLevel::H1 => 1,
1094        HeadingLevel::H2 => 2,
1095        HeadingLevel::H3 => 3,
1096        HeadingLevel::H4 => 4,
1097        HeadingLevel::H5 => 5,
1098        HeadingLevel::H6 => 6,
1099    }
1100}
1101
1102// ─── HTML parsing ────────────────────────────────────────────────────
1103
1104use scraper::Node;
1105
1106/// Parsed CSS block-level styles from an inline `style` attribute.
1107#[derive(Debug, Clone, Default)]
1108struct BlockStyles {
1109    line_height: Option<i64>,
1110    non_breakable_lines: Option<bool>,
1111    page_break_before: Option<bool>,
1112    direction: Option<TextDirection>,
1113    background_color: Option<String>,
1114    /// `white-space` asks for the source's own spacing to be kept verbatim.
1115    /// Separate from `non_breakable_lines` because the two do not coincide:
1116    /// `nowrap` forbids wrapping but still collapses runs of spaces, so a
1117    /// block carrying it must be normalised like any other.
1118    preserve_whitespace: Option<bool>,
1119}
1120
1121/// Parse relevant CSS properties from an inline style string.
1122/// Handles: line-height, white-space, break-before/page-break-before, direction,
1123/// background-color.
1124fn parse_block_styles(style: &str) -> BlockStyles {
1125    let mut result = BlockStyles::default();
1126    for part in style.split(';') {
1127        let part = part.trim();
1128        if let Some((prop, val)) = part.split_once(':') {
1129            let prop = prop.trim().to_ascii_lowercase();
1130            let val = val.trim();
1131            match prop.as_str() {
1132                "line-height" => {
1133                    // Try parsing as a plain number (multiplier)
1134                    if let Ok(v) = val.parse::<f64>() {
1135                        result.line_height = Some((v * 1000.0) as i64);
1136                    }
1137                }
1138                "white-space" if val == "pre" || val == "nowrap" || val == "pre-wrap" => {
1139                    result.non_breakable_lines = Some(true);
1140                    if val != "nowrap" {
1141                        result.preserve_whitespace = Some(true);
1142                    }
1143                }
1144                // CSS3 `break-before` and its CSS2 predecessor `page-break-before` mean
1145                // the same thing; both are read because both are written (browsers still
1146                // want the legacy spelling, and so do most EPUB engines).
1147                "break-before" | "page-break-before" => {
1148                    result.page_break_before = match val.to_ascii_lowercase().as_str() {
1149                        "page" | "always" | "left" | "right" | "recto" | "verso" => Some(true),
1150                        "avoid" | "auto" => Some(false),
1151                        _ => None,
1152                    };
1153                }
1154                "direction" => {
1155                    if val.eq_ignore_ascii_case("rtl") {
1156                        result.direction = Some(TextDirection::RightToLeft);
1157                    } else if val.eq_ignore_ascii_case("ltr") {
1158                        result.direction = Some(TextDirection::LeftToRight);
1159                    }
1160                }
1161                "background-color" | "background" => {
1162                    result.background_color = Some(val.to_string());
1163                }
1164                _ => {}
1165            }
1166        }
1167    }
1168    result
1169}
1170
1171/// The five characters HTML calls whitespace.
1172///
1173/// Deliberately not `char::is_whitespace`, which is `true` for U+00A0: a
1174/// no-break space is a character the writer typed — the one French typography
1175/// puts before a colon, holding "Attention&nbsp;:" together — and collapsing it
1176/// away is a silent edit of the text, not of its layout.
1177fn is_html_space(ch: char) -> bool {
1178    matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{0C}')
1179}
1180
1181/// Whether a span run holds anything a reader would see.
1182fn spans_carry_content(spans: &[ParsedSpan]) -> bool {
1183    spans
1184        .iter()
1185        .any(|s| !s.text.is_empty() || s.image.is_some() || s.footnote_ref.is_some())
1186}
1187
1188/// Apply the CSS `white-space: normal` rules to one block's inline spans: every
1189/// run of HTML whitespace collapses to a single space, and the runs at the
1190/// block's two ends disappear entirely.
1191///
1192/// Per block rather than per text node, because a run crosses span boundaries —
1193/// `<b>word</b>\n<i>next</i>` is one collapsible run split over three nodes —
1194/// and because only the block knows where its own ends are.
1195///
1196/// Without this, an exporter's own source formatting reaches the document as
1197/// text. LibreOffice writes a newline after every `<p …>` and hard-wraps the
1198/// prose inside it at about seventy columns, so pasting a whole book put a
1199/// blank line before every paragraph and a break through the middle of every
1200/// sentence.
1201fn collapse_inline_whitespace(spans: &mut [ParsedSpan]) {
1202    fn is_replaced(span: &ParsedSpan) -> bool {
1203        span.image.is_some() || span.footnote_ref.is_some()
1204    }
1205
1206    // Collapse within each span first. Nothing non-empty is emptied here, which
1207    // is what lets the second pass read `is_empty()` as "this span held no text
1208    // to begin with" — the marker a `<br>` leaves behind.
1209    for span in spans.iter_mut() {
1210        if is_replaced(span) {
1211            continue;
1212        }
1213        let mut out = String::with_capacity(span.text.len());
1214        let mut in_run = false;
1215        for ch in span.text.chars() {
1216            if is_html_space(ch) {
1217                if !in_run {
1218                    out.push(' ');
1219                    in_run = true;
1220                }
1221            } else {
1222                out.push(ch);
1223                in_run = false;
1224            }
1225        }
1226        span.text = out;
1227    }
1228
1229    // The block's start counts as "already ended on a space", so a run opening
1230    // the block is dropped outright.
1231    let mut prev_ends_with_space = true;
1232    for span in spans.iter_mut() {
1233        if is_replaced(span) {
1234            prev_ends_with_space = false;
1235            continue;
1236        }
1237        if span.text.is_empty() {
1238            // The empty span a `<br>` leaves. It contributes no text, but it
1239            // still ends the collapsible run either side of it — without that,
1240            // `x <br> y` closes up into "xy".
1241            prev_ends_with_space = false;
1242            continue;
1243        }
1244        if prev_ends_with_space && span.text.starts_with(' ') {
1245            span.text.remove(0);
1246        }
1247        // A span emptied by that drop leaves the run open rather than closing
1248        // it: the space it carried belongs to the run its neighbours share.
1249        if !span.text.is_empty() {
1250            prev_ends_with_space = span.text.ends_with(' ');
1251        }
1252    }
1253
1254    // The block's trailing run, in whichever span still holds it. Replaced
1255    // content stops the walk: a space before a trailing image is between two
1256    // pieces of content, not after the last one.
1257    for span in spans.iter_mut().rev() {
1258        if is_replaced(span) {
1259            break;
1260        }
1261        if span.text.is_empty() {
1262            continue;
1263        }
1264        if span.text.ends_with(' ') {
1265            span.text.pop();
1266        }
1267        break;
1268    }
1269}
1270
1271/// `block` as one block per line of its text.
1272///
1273/// The model has no line break inside a block: a new line is a new block, which is
1274/// what the Djot reader makes of a hard break and what a plain-text paste makes of a
1275/// line break. Text that kept its source's line breaks (`<pre>`, `white-space: pre`)
1276/// held them inside one block, where the editor showed them as lines and the Djot
1277/// writer had nothing to write them as. Written raw, a blank line ended the paragraph
1278/// and the lines after it were read back as the lists, headings and quotations their
1279/// markers spelled, however deep.
1280///
1281/// Each line keeps the block's own formatting and its spans' formatting, but for the
1282/// page break, which stays with the first line: it is where the block starts, and on
1283/// every line it put each on a page of its own. An empty line is an empty block. The
1284/// line break that ends the last line starts no line of its own, as a browser shows it.
1285/// A block without a line break is returned as it is.
1286///
1287/// Every line is built from copies of the block and of the span it comes from that
1288/// hold no text, so the split costs in proportion to the text: built from whole copies,
1289/// each line copied the whole text again, and a paste of 64,000 preformatted lines
1290/// took seconds.
1291pub fn split_block_at_line_breaks(mut block: ParsedBlock) -> Vec<ParsedBlock> {
1292    let breaks = |span: &ParsedSpan| span.image.is_none() && span.text.contains('\n');
1293    if !block.spans.iter().any(breaks) {
1294        return vec![block];
1295    }
1296    let mut lines: Vec<Vec<ParsedSpan>> = vec![Vec::new()];
1297    for mut span in std::mem::take(&mut block.spans) {
1298        if !breaks(&span) {
1299            push_to_last_line(&mut lines, span);
1300            continue;
1301        }
1302        let text = std::mem::take(&mut span.text);
1303        for (i, piece) in text.split('\n').enumerate() {
1304            if i > 0 {
1305                lines.push(Vec::new());
1306            }
1307            if !piece.is_empty() {
1308                push_to_last_line(
1309                    &mut lines,
1310                    ParsedSpan {
1311                        text: piece.to_string(),
1312                        ..span.clone()
1313                    },
1314                );
1315            }
1316        }
1317    }
1318    if lines.len() > 1 && lines.last().is_some_and(Vec::is_empty) {
1319        lines.pop();
1320    }
1321    let later_line = ParsedBlock {
1322        page_break_before: None,
1323        blockquote_opens: 0,
1324        ..block.clone()
1325    };
1326    let mut first_line = Some(block);
1327    lines
1328        .into_iter()
1329        .map(|spans| {
1330            let spans = if spans.is_empty() {
1331                vec![ParsedSpan::default()]
1332            } else {
1333                spans
1334            };
1335            match first_line.take() {
1336                Some(first) => ParsedBlock { spans, ..first },
1337                None => ParsedBlock {
1338                    spans,
1339                    ..later_line.clone()
1340                },
1341            }
1342        })
1343        .collect()
1344}
1345
1346/// Append `span` to the last of `lines`, which [`split_block_at_line_breaks`] keeps
1347/// non-empty.
1348fn push_to_last_line(lines: &mut Vec<Vec<ParsedSpan>>, span: ParsedSpan) {
1349    match lines.last_mut() {
1350        Some(line) => line.push(span),
1351        None => lines.push(vec![span]),
1352    }
1353}
1354
1355pub fn parse_html(html: &str) -> Vec<ParsedBlock> {
1356    ParsedElement::flatten_to_blocks(parse_html_elements(html))
1357}
1358
1359/// The attribute an HTML producer uses to say "this element *is* a footnote
1360/// reference", and the label it names.
1361///
1362/// A data attribute on an otherwise-ordinary element, rather than a literal
1363/// `[^label]` in the text, because the text a producer emits is *escaped* on the
1364/// way to Djot — [`escape_djot_inline`](crate::parser_tools::djot_escape::escape_djot_inline)
1365/// neutralises `[`, `]` and `^`, so a reference smuggled through as characters
1366/// arrives as `\[\^1\]` and is prose rather than a reference. It has to be a
1367/// node before it is serialised, which is what this makes it.
1368///
1369/// [`ParsedSpan::footnote_ref`] is the same field the Markdown/Djot reader
1370/// already fills, so everything downstream — the document model, the Djot
1371/// writer, numbering — needs no change at all.
1372pub const HTML_FOOTNOTE_ATTR: &str = "data-footnote-ref";
1373
1374/// Build a footnote-reference span from any element carrying
1375/// [`HTML_FOOTNOTE_ATTR`].
1376fn html_footnote_span(
1377    el: &scraper::node::Element,
1378    link_href: Option<String>,
1379) -> Option<ParsedSpan> {
1380    let label = el.attr(HTML_FOOTNOTE_ATTR)?.trim();
1381    if label.is_empty() {
1382        return None;
1383    }
1384    Some(ParsedSpan {
1385        text: String::new(),
1386        link_href,
1387        footnote_ref: Some(label.to_string()),
1388        ..Default::default()
1389    })
1390}
1391
1392/// Build an inline-image span from an `<img>` element, if it has a usable
1393/// source.
1394///
1395/// `<img>` was matched by none of the HTML walker's three tag dispatches, so it
1396/// fell into their wildcard arms and was dropped whole — silently, and unlike
1397/// Markdown not even leaving its alt text behind. That is the path a browser or
1398/// Word paste travels.
1399fn html_img_span(el: &scraper::node::Element, link_href: Option<String>) -> Option<ParsedSpan> {
1400    let src = el.attr("src")?;
1401    if src.is_empty() {
1402        return None;
1403    }
1404    let dim = |name: &str| -> i64 {
1405        el.attr(name)
1406            .and_then(|v| v.trim().trim_end_matches("px").parse::<i64>().ok())
1407            .filter(|n| *n > 0)
1408            .unwrap_or(0)
1409    };
1410    Some(ParsedSpan {
1411        text: String::new(),
1412        link_href,
1413        image: Some(ParsedImage {
1414            src: src.to_string(),
1415            alt: el.attr("alt").unwrap_or_default().to_string(),
1416            width: dim("width"),
1417            height: dim("height"),
1418        }),
1419        ..Default::default()
1420    })
1421}
1422
1423/// Whether `node` is a `<ul>` or an `<ol>`.
1424fn is_html_list(node: &Node) -> bool {
1425    node.as_element()
1426        .is_some_and(|el| matches!(el.name(), "ul" | "ol"))
1427}
1428
1429/// Move every list item nested deeper than
1430/// [`MAX_LIST_LEVELS`](crate::parser_tools::list_depth::MAX_LIST_LEVELS)
1431/// lists into the deepest list kept, right after the item holding it, in
1432/// document order: a list nested past that depth comes out flattened to it,
1433/// every item kept, in its order.
1434///
1435/// The reader recurses a few elements deep per list level and stops at
1436/// `MAX_RECURSION_DEPTH`, so a list nested past some eighty levels lost every
1437/// item below that, pasted or loaded. The editing gestures nest no deeper than
1438/// this depth, and an insertion brings any list back to it (see `list_depth`),
1439/// so HTML holding more is read at that depth to begin with. The tree is
1440/// changed in place, without recursion, before it is read.
1441fn flatten_deep_html_lists(tree: &mut ego_tree::Tree<Node>) {
1442    use crate::parser_tools::list_depth::MAX_LIST_LEVELS;
1443    use ego_tree::iter::Edge;
1444
1445    // The lists one level past the deepest kept, in document order.
1446    let mut too_deep: Vec<ego_tree::NodeId> = Vec::new();
1447    let mut depth = 0usize;
1448    for edge in tree.root().traverse() {
1449        match edge {
1450            Edge::Open(node) if is_html_list(node.value()) => {
1451                if depth == MAX_LIST_LEVELS {
1452                    too_deep.push(node.id());
1453                }
1454                depth += 1;
1455            }
1456            Edge::Close(node) if is_html_list(node.value()) => {
1457                depth = depth.saturating_sub(1);
1458            }
1459            _ => {}
1460        }
1461    }
1462
1463    for list in too_deep {
1464        // The child of the deepest kept list that holds `list` (its `<li>`, or
1465        // `list` itself when a list sits right inside another): the items go
1466        // after it.
1467        let Some(list_ref) = tree.get(list) else {
1468            continue;
1469        };
1470        let mut holder = None;
1471        let mut current = list_ref;
1472        while let Some(parent) = current.parent() {
1473            if is_html_list(parent.value()) {
1474                holder = Some(current.id());
1475                break;
1476            }
1477            current = parent;
1478        }
1479        let Some(holder) = holder else {
1480            continue;
1481        };
1482        let mut after = holder;
1483        let items: Vec<ego_tree::NodeId> = list_ref
1484            .descendants()
1485            .filter(|node| {
1486                node.value()
1487                    .as_element()
1488                    .is_some_and(|el| el.name() == "li")
1489            })
1490            .map(|node| node.id())
1491            .collect();
1492        for item in items {
1493            // Each node placed here sits in the deepest kept list, so it has a
1494            // parent to be placed beside.
1495            if tree.get(after).and_then(|node| node.parent()).is_none() {
1496                break;
1497            }
1498            if let Some(mut node) = tree.get_mut(after) {
1499                node.insert_id_after(item);
1500                after = item;
1501            }
1502        }
1503        // What is left of the list is its items' wrappers, or text standing in
1504        // it outside any item. It goes after the items when it holds any text
1505        // (it is already beside them when it sits right in the deepest kept
1506        // list), and away when it holds none.
1507        let holds_text = tree.get(list).is_some_and(|node| {
1508            node.descendants()
1509                .any(|d| d.value().as_text().is_some_and(|t| !t.trim().is_empty()))
1510        });
1511        if !holds_text {
1512            if let Some(mut node) = tree.get_mut(list) {
1513                node.detach();
1514            }
1515        } else if holder != list
1516            && tree.get(after).and_then(|node| node.parent()).is_some()
1517            && let Some(mut node) = tree.get_mut(after)
1518        {
1519            node.insert_id_after(list);
1520        }
1521    }
1522}
1523
1524/// How deep a walk of an HTML tree may go on the caller's own stack: a debug build spends
1525/// between 5 and 8 KiB of stack on each element level it follows, so this is at most some
1526/// 256 KiB, what any thread a host reads on has to spare.
1527const HTML_DEPTH_ON_THE_CALLERS_STACK: usize = 32;
1528
1529/// The deepest an HTML reader follows elements: a subtree nested deeper is read as its
1530/// text, without recursing (see `read_html_tree`).
1531const MAX_HTML_READ_DEPTH: usize = 256;
1532
1533/// The stack a reader of a deeper tree gets on a thread of its own, before what each level
1534/// adds: a spawned thread's.
1535const HTML_READER_STACK: usize = 2 << 20;
1536
1537/// The stack each element level of a deeper tree adds for its reader: two to three times
1538/// what a debug build spends on one.
1539const HTML_STACK_PER_LEVEL: usize = 16 << 10;
1540
1541/// `value` read as HTML reads an integer attribute such as `<ol start>`: after any leading
1542/// white space, an optional sign and the digits up to the first character that is not one.
1543/// `None` when no digit follows. A number past what 64 bits hold reads as the largest they
1544/// hold on its side of 0. A browser numbers `<ol start="5x">` from 5, where a strict parse
1545/// of the whole value numbered it from 1.
1546fn html_integer(value: &str) -> Option<i64> {
1547    let value = value.trim_start_matches(|c: char| c.is_ascii_whitespace());
1548    let (negative, digits) = match value.as_bytes().first() {
1549        Some(b'-') => (true, &value[1..]),
1550        Some(b'+') => (false, &value[1..]),
1551        _ => (false, value),
1552    };
1553    let digits: &str = digits
1554        .split(|c: char| !c.is_ascii_digit())
1555        .next()
1556        .unwrap_or("");
1557    if digits.is_empty() {
1558        return None;
1559    }
1560    let magnitude = digits.bytes().fold(0i64, |number, digit| {
1561        number
1562            .saturating_mul(10)
1563            .saturating_add(i64::from(digit - b'0'))
1564    });
1565    Some(if negative { -magnitude } else { magnitude })
1566}
1567
1568/// How many elements deep the deepest node of `tree` is. A loop over the tree's edges,
1569/// never a recursion.
1570fn html_tree_depth(tree: &ego_tree::Tree<Node>) -> usize {
1571    use ego_tree::iter::Edge;
1572    let mut depth = 0usize;
1573    let mut deepest = 0usize;
1574    for edge in tree.root().traverse() {
1575        match edge {
1576            Edge::Open(node) if node.value().is_element() => {
1577                depth += 1;
1578                deepest = deepest.max(depth);
1579            }
1580            Edge::Close(node) if node.value().is_element() => depth = depth.saturating_sub(1),
1581            _ => {}
1582        }
1583    }
1584    deepest
1585}
1586
1587/// Read `html` into blocks and tables.
1588///
1589/// The reader recurses once per element level it follows, up to `MAX_HTML_READ_DEPTH`,
1590/// and a stack overflow aborts the process: a debug build ran out of a spawned thread's
1591/// stack on a paste nested some 200 elements deep, and a host reading on a smaller thread
1592/// sooner. A tree nested deeper than a caller's stack is sure to hold is read on a thread
1593/// of its own, with a stack sized for its depth; when no thread can be started, it is read
1594/// on the caller's stack to `HTML_DEPTH_ON_THE_CALLERS_STACK` levels, each subtree
1595/// deeper than that read as its text. The quotations it holds are kept within
1596/// [`MAX_QUOTE_LEVELS`](crate::parser_tools::quote_depth::MAX_QUOTE_LEVELS) (see
1597/// `quote_depth`).
1598pub fn parse_html_elements(html: &str) -> Vec<ParsedElement> {
1599    use scraper::Html;
1600
1601    let read = |depth_limit: usize, html: &str| {
1602        let mut fragment = Html::parse_fragment(html);
1603        flatten_deep_html_lists(&mut fragment.tree);
1604        let mut elements = read_html_tree(&fragment, depth_limit);
1605        crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1606        elements
1607    };
1608    let mut fragment = Html::parse_fragment(html);
1609    flatten_deep_html_lists(&mut fragment.tree);
1610    let depth = html_tree_depth(&fragment.tree);
1611    if depth <= HTML_DEPTH_ON_THE_CALLERS_STACK {
1612        let mut elements = read_html_tree(&fragment, MAX_HTML_READ_DEPTH);
1613        crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1614        return elements;
1615    }
1616    drop(fragment);
1617    let stack = HTML_READER_STACK + depth.min(MAX_HTML_READ_DEPTH + 1) * HTML_STACK_PER_LEVEL;
1618    let owned = html.to_owned();
1619    let on_its_own_thread = std::thread::Builder::new()
1620        .stack_size(stack)
1621        .spawn(move || read(MAX_HTML_READ_DEPTH, &owned))
1622        .ok()
1623        .and_then(|reader| reader.join().ok());
1624    on_its_own_thread.unwrap_or_else(|| read(HTML_DEPTH_ON_THE_CALLERS_STACK, html))
1625}
1626
1627/// Read `fragment`, following its elements `depth_limit` deep at most: a subtree nested
1628/// deeper is read as its text (see `subtree_text`). The walk recurses once per element
1629/// level it follows.
1630fn read_html_tree(fragment: &scraper::Html, depth_limit: usize) -> Vec<ParsedElement> {
1631    let mut elements: Vec<ParsedElement> = Vec::new();
1632
1633    // Walk the DOM tree starting from the root
1634    let root = fragment.root_element();
1635
1636    #[derive(Clone, Default)]
1637    struct FmtState {
1638        bold: bool,
1639        italic: bool,
1640        underline: bool,
1641        strikeout: bool,
1642        code: bool,
1643        superscript: bool,
1644        subscript: bool,
1645        link_href: Option<String>,
1646    }
1647
1648    /// The list an element stands in: its style, and the number its first item wears when
1649    /// it is not 1 (`<ol start="3">`).
1650    type HtmlList = (Option<ListStyle>, Option<i64>);
1651
1652    /// Elements whose text content is machinery, not prose.
1653    ///
1654    /// `<style>` and `<script>` hold raw text the HTML spec never renders, and
1655    /// `<head>` and its metadata children carry none a reader is meant to see.
1656    /// Every walker below recurses into unknown tags on the assumption that
1657    /// they are inline wrappers, which turns a stylesheet into paragraphs —
1658    /// and Word, Google Docs and Chrome all put a `<style>` block in the
1659    /// `text/html` flavour they publish to the clipboard, so this is the
1660    /// ordinary paste, not an exotic one.
1661    fn is_metadata_tag(tag: &str) -> bool {
1662        matches!(
1663            tag,
1664            "head"
1665                | "style"
1666                | "script"
1667                | "title"
1668                | "meta"
1669                | "link"
1670                | "base"
1671                | "noscript"
1672                | "template"
1673        )
1674    }
1675
1676    /// The text of everything under `node`, for a subtree nested deeper than
1677    /// the reader follows (see [`MAX_HTML_READ_DEPTH`]): read without recursing,
1678    /// with a space where a block ends so that words do not run together, and
1679    /// without what [`is_metadata_tag`] names.
1680    ///
1681    /// Past that depth the reader used to stop and keep nothing: text nested
1682    /// deeply enough in lists, quotations or plain `<div>`s was dropped from a
1683    /// paste without a trace.
1684    fn subtree_text(node: ego_tree::NodeRef<Node>) -> String {
1685        use ego_tree::iter::Edge;
1686        let mut text = String::new();
1687        // How many elements deep the walk is inside a metadata element.
1688        let mut hidden = 0usize;
1689        for edge in node.traverse() {
1690            match edge {
1691                Edge::Open(open) => match open.value() {
1692                    Node::Element(el) => {
1693                        if hidden > 0 || is_metadata_tag(el.name()) {
1694                            hidden += 1;
1695                        } else if !text.is_empty() && !text.ends_with(char::is_whitespace) {
1696                            text.push(' ');
1697                        }
1698                    }
1699                    Node::Text(t) if hidden == 0 => text.push_str(t),
1700                    _ => {}
1701                },
1702                Edge::Close(close) => {
1703                    if hidden > 0 && close.value().is_element() {
1704                        hidden -= 1;
1705                    }
1706                }
1707            }
1708        }
1709        text
1710    }
1711
1712    /// A span of `text` in `state`'s formatting.
1713    fn text_span(text: String, state: &FmtState) -> ParsedSpan {
1714        ParsedSpan {
1715            text,
1716            bold: state.bold,
1717            italic: state.italic,
1718            underline: state.underline,
1719            strikeout: state.strikeout,
1720            code: state.code,
1721            superscript: state.superscript,
1722            subscript: state.subscript,
1723            link_href: state.link_href.clone(),
1724            image: None,
1725            footnote_ref: None,
1726        }
1727    }
1728
1729    /// Collect inline spans from a `<td>` or `<th>` cell element.
1730    fn collect_cell_spans(
1731        node: ego_tree::NodeRef<Node>,
1732        state: &FmtState,
1733        spans: &mut Vec<ParsedSpan>,
1734        depth: usize,
1735        limit: usize,
1736    ) {
1737        if depth > limit {
1738            let text = subtree_text(node);
1739            if !text.is_empty() {
1740                spans.push(text_span(text, state));
1741            }
1742            return;
1743        }
1744        for child in node.children() {
1745            match child.value() {
1746                Node::Text(text) => {
1747                    let t = text.text.to_string();
1748                    if !t.is_empty() {
1749                        spans.push(ParsedSpan {
1750                            text: t,
1751                            bold: state.bold,
1752                            italic: state.italic,
1753                            underline: state.underline,
1754                            strikeout: state.strikeout,
1755                            code: state.code,
1756                            superscript: state.superscript,
1757                            subscript: state.subscript,
1758                            link_href: state.link_href.clone(),
1759                            image: None,
1760                            footnote_ref: None,
1761                        });
1762                    }
1763                }
1764                Node::Element(el) => {
1765                    let tag = el.name();
1766                    if is_metadata_tag(tag) {
1767                        continue;
1768                    }
1769                    let mut new_state = state.clone();
1770                    match tag {
1771                        // Checked by attribute rather than by tag name: the
1772                        // producer picks the element (a `<sup>` renders sensibly
1773                        // in a browser), and only the attribute is a contract.
1774                        _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
1775                            if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
1776                            {
1777                                spans.push(span);
1778                            }
1779                            continue;
1780                        }
1781                        "b" | "strong" => new_state.bold = true,
1782                        "i" | "em" => new_state.italic = true,
1783                        "u" | "ins" => new_state.underline = true,
1784                        "s" | "del" | "strike" => new_state.strikeout = true,
1785                        "code" => new_state.code = true,
1786                        "sup" => new_state.superscript = true,
1787                        "sub" => new_state.subscript = true,
1788                        "a" => {
1789                            if let Some(href) = el.attr("href") {
1790                                new_state.link_href = Some(href.to_string());
1791                            }
1792                        }
1793                        "img" => {
1794                            if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
1795                                spans.push(span);
1796                            }
1797                            continue;
1798                        }
1799                        // A cell is read as one paragraph, so a line break, or a
1800                        // paragraph or list item of its own, is a space between
1801                        // words. Read as nothing, a cell of two paragraphs, which
1802                        // the HTML writer puts out as `One<br/>Two`, came back as
1803                        // the one word `OneTwo`.
1804                        "br" => {
1805                            spans.push(text_span(" ".to_string(), state));
1806                            continue;
1807                        }
1808                        "p" | "div" | "li" | "ul" | "ol" | "blockquote" | "pre" | "h1" | "h2"
1809                        | "h3" | "h4" | "h5" | "h6" => {
1810                            spans.push(text_span(" ".to_string(), state));
1811                        }
1812                        _ => {}
1813                    }
1814                    collect_cell_spans(child, &new_state, spans, depth + 1, limit);
1815                }
1816                _ => {}
1817            }
1818        }
1819    }
1820
1821    /// The widest a `colspan` makes a row: HTML's own limit on the attribute. A row of
1822    /// ordinary cells may be wider; only the empty cells a span adds stop here.
1823    const MAX_HTML_COLSPAN: usize = 1000;
1824
1825    /// Parse a `<table>` element into a ParsedTable, with the number of `<td>` and `<th>`
1826    /// cells it has (see `push_table`). `depth` is how deep the walk stands at the table,
1827    /// and `limit` how deep it goes.
1828    fn parse_table_element(
1829        table_node: ego_tree::NodeRef<Node>,
1830        depth: usize,
1831        limit: usize,
1832    ) -> (ParsedTable, usize) {
1833        let mut rows: Vec<Vec<ParsedTableCell>> = Vec::new();
1834        let mut header_rows: usize = 0;
1835        let mut own_cells: usize = 0;
1836
1837        /// `walk` is how deep the walk stands at `node` and how deep it goes.
1838        fn collect_rows(
1839            node: ego_tree::NodeRef<Node>,
1840            rows: &mut Vec<Vec<ParsedTableCell>>,
1841            header_rows: &mut usize,
1842            own_cells: &mut usize,
1843            in_thead: bool,
1844            walk: (usize, usize),
1845        ) {
1846            let (depth, limit) = walk;
1847            for child in node.children() {
1848                if let Node::Element(el) = child.value() {
1849                    match el.name() {
1850                        "thead" => collect_rows(
1851                            child,
1852                            rows,
1853                            header_rows,
1854                            own_cells,
1855                            true,
1856                            (depth + 1, limit),
1857                        ),
1858                        "tbody" | "tfoot" => collect_rows(
1859                            child,
1860                            rows,
1861                            header_rows,
1862                            own_cells,
1863                            false,
1864                            (depth + 1, limit),
1865                        ),
1866                        "tr" => {
1867                            let mut cells: Vec<ParsedTableCell> = Vec::new();
1868                            // The cells of the row up to its last `<td>` or `<th>`: the
1869                            // empty cells a span adds after it hold no cell's place.
1870                            let mut own_end = 0;
1871                            for td in child.children() {
1872                                if let Node::Element(td_el) = td.value()
1873                                    && matches!(td_el.name(), "td" | "th")
1874                                {
1875                                    let mut spans = Vec::new();
1876                                    let state = FmtState::default();
1877                                    collect_cell_spans(td, &state, &mut spans, depth + 2, limit);
1878                                    collapse_inline_whitespace(&mut spans);
1879                                    if spans.is_empty() {
1880                                        spans.push(ParsedSpan::default());
1881                                    }
1882                                    cells.push(ParsedTableCell { spans });
1883                                    *own_cells += 1;
1884                                    own_end = cells.len();
1885                                    // A cell spanning columns covers the cells after it
1886                                    // in its row: empty ones hold its place, so the cells
1887                                    // that follow stay in their own columns and the
1888                                    // table is as wide as the columns it spans.
1889                                    let span = td_el
1890                                        .attr("colspan")
1891                                        .and_then(|value| value.trim().parse::<usize>().ok())
1892                                        .unwrap_or(1)
1893                                        .clamp(1, MAX_HTML_COLSPAN);
1894                                    for _ in 1..span {
1895                                        if cells.len() >= MAX_HTML_COLSPAN {
1896                                            break;
1897                                        }
1898                                        cells.push(ParsedTableCell {
1899                                            spans: vec![ParsedSpan::default()],
1900                                        });
1901                                    }
1902                                }
1903                            }
1904                            // A span closing a row widens the table only as far as another
1905                            // row's cells reach: `<th colspan="1000">` over rows of one
1906                            // cell is a table of one column, not of a thousand.
1907                            cells.truncate(own_end);
1908                            if !cells.is_empty() {
1909                                rows.push(cells);
1910                                if in_thead {
1911                                    *header_rows += 1;
1912                                }
1913                            }
1914                        }
1915                        _ => {}
1916                    }
1917                }
1918            }
1919        }
1920
1921        collect_rows(
1922            table_node,
1923            &mut rows,
1924            &mut header_rows,
1925            &mut own_cells,
1926            false,
1927            (depth, limit),
1928        );
1929
1930        // Tables without explicit <thead> but with <th> cells: treat first row as header
1931        if header_rows == 0 && !rows.is_empty() {
1932            header_rows = 1;
1933        }
1934
1935        (
1936            ParsedTable {
1937                header_rows,
1938                rows,
1939                // The caller (`walk_node`) sets the real depth: this helper has
1940                // no visibility into the surrounding blockquote nesting.
1941                blockquote_depth: 0,
1942                blockquote_opens: 0,
1943            },
1944            own_cells,
1945        )
1946    }
1947
1948    #[allow(clippy::too_many_arguments)]
1949    fn walk_node(
1950        node: ego_tree::NodeRef<Node>,
1951        state: &FmtState,
1952        elements: &mut Vec<ParsedElement>,
1953        current_list: &HtmlList,
1954        blockquote_depth: u32,
1955        list_depth: u32,
1956        depth: usize,
1957        limit: usize,
1958    ) {
1959        if depth > limit {
1960            let text = subtree_text(node);
1961            if !text.trim().is_empty() {
1962                let mut spans = vec![text_span(text, state)];
1963                collapse_inline_whitespace(&mut spans);
1964                elements.push(ParsedElement::Block(ParsedBlock {
1965                    spans,
1966                    blockquote_depth,
1967                    blockquote_opens: 0,
1968                    ..ParsedBlock::default()
1969                }));
1970            }
1971            return;
1972        }
1973        match node.value() {
1974            Node::Element(el) => {
1975                let tag = el.name();
1976                if is_metadata_tag(tag) {
1977                    return;
1978                }
1979                let mut new_state = state.clone();
1980                let mut new_list = current_list.clone();
1981                let mut bq_depth = blockquote_depth;
1982                let mut new_list_depth = list_depth;
1983
1984                // Determine if this is a block-level element
1985                let is_block_tag = matches!(
1986                    tag,
1987                    "p" | "div"
1988                        | "h1"
1989                        | "h2"
1990                        | "h3"
1991                        | "h4"
1992                        | "h5"
1993                        | "h6"
1994                        | "li"
1995                        | "pre"
1996                        | "br"
1997                        | "blockquote"
1998                        | "body"
1999                        | "html"
2000                );
2001
2002                // Update formatting state
2003                match tag {
2004                    "b" | "strong" => new_state.bold = true,
2005                    "i" | "em" => new_state.italic = true,
2006                    "u" | "ins" => new_state.underline = true,
2007                    "s" | "del" | "strike" => new_state.strikeout = true,
2008                    "code" => new_state.code = true,
2009                    "sup" => new_state.superscript = true,
2010                    "sub" => new_state.subscript = true,
2011                    "a" => {
2012                        if let Some(href) = el.attr("href") {
2013                            new_state.link_href = Some(href.to_string());
2014                        }
2015                    }
2016                    "ul" => {
2017                        new_list = (Some(ListStyle::Disc), None);
2018                        new_list_depth = list_depth + 1;
2019                    }
2020                    "ol" => {
2021                        // A list numbered from below 0 is kept from 1: no saved syntax
2022                        // but HTML writes one. A start past what every format holds is
2023                        // read as the largest it holds (see `list_start`).
2024                        let start = el
2025                            .attr("start")
2026                            .and_then(html_integer)
2027                            .and_then(crate::parser_tools::list_start::list_start);
2028                        new_list = (Some(ListStyle::Decimal), start);
2029                        new_list_depth = list_depth + 1;
2030                    }
2031                    "blockquote" => {
2032                        bq_depth += 1;
2033                    }
2034                    _ => {}
2035                }
2036
2037                // Determine heading level
2038                let heading_level = match tag {
2039                    "h1" => Some(1),
2040                    "h2" => Some(2),
2041                    "h3" => Some(3),
2042                    "h4" => Some(4),
2043                    "h5" => Some(5),
2044                    "h6" => Some(6),
2045                    _ => None,
2046                };
2047
2048                let is_code_block = tag == "pre";
2049
2050                // Extract code language from <pre><code class="language-xxx">
2051                let code_language = if is_code_block {
2052                    node.children().find_map(|child| {
2053                        if let Node::Element(cel) = child.value()
2054                            && cel.name() == "code"
2055                            && let Some(cls) = cel.attr("class")
2056                        {
2057                            return cls
2058                                .split_whitespace()
2059                                .find_map(|c| c.strip_prefix("language-"))
2060                                .map(|l| l.to_string());
2061                        }
2062                        None
2063                    })
2064                } else {
2065                    None
2066                };
2067
2068                // Extract CSS styles from block-level elements
2069                let css = if is_block_tag {
2070                    el.attr("style").map(parse_block_styles).unwrap_or_default()
2071                } else {
2072                    BlockStyles::default()
2073                };
2074
2075                if tag == "table" {
2076                    // Parse table structure into a ParsedTable
2077                    let (mut parsed_table, own_cells) = parse_table_element(node, depth, limit);
2078                    if !parsed_table.rows.is_empty() {
2079                        parsed_table.blockquote_depth = bq_depth;
2080                        push_table(elements, parsed_table, own_cells);
2081                    }
2082                    return;
2083                }
2084
2085                if tag == "br" {
2086                    // <br> creates a new block
2087                    elements.push(ParsedElement::Block(ParsedBlock {
2088                        spans: vec![ParsedSpan {
2089                            text: String::new(),
2090                            ..Default::default()
2091                        }],
2092                        heading_level: None,
2093                        list_style: None,
2094                        list_indent: 0,
2095                        list_prefix: String::new(),
2096                        list_suffix: String::new(),
2097                        list_start: None,
2098                        marker: None,
2099                        is_code_block: false,
2100                        code_language: None,
2101                        blockquote_depth: bq_depth,
2102                        blockquote_opens: 0,
2103                        line_height: None,
2104                        non_breakable_lines: None,
2105                        page_break_before: None,
2106                        front_matter: None,
2107                        direction: None,
2108                        background_color: None,
2109                        alignment: None,
2110                        top_margin: None,
2111                        text_indent: None,
2112                        semantic_role: None,
2113                    }));
2114                    return;
2115                }
2116
2117                if tag == "blockquote" {
2118                    // Blockquote is a container — recurse into children with increased depth
2119                    let first = elements.len();
2120                    for child in node.children() {
2121                        walk_node(
2122                            child,
2123                            &new_state,
2124                            elements,
2125                            &new_list,
2126                            bq_depth,
2127                            new_list_depth,
2128                            depth + 1,
2129                            limit,
2130                        );
2131                    }
2132                    mark_quotation_opened(elements, first);
2133                } else if is_block_tag && tag != "br" {
2134                    // Start collecting spans for a new block.
2135                    // Use a temporary buffer so that nested block-level
2136                    // elements (e.g. sub-lists inside <li>) are collected
2137                    // separately and appended *after* the parent block.
2138                    let mut spans: Vec<ParsedSpan> = Vec::new();
2139                    let mut nested_elements: Vec<ParsedElement> = Vec::new();
2140                    collect_inline_spans(
2141                        node,
2142                        &new_state,
2143                        &mut spans,
2144                        &new_list,
2145                        &mut nested_elements,
2146                        bq_depth,
2147                        new_list_depth,
2148                        depth + 1,
2149                        limit,
2150                    );
2151
2152                    let (list_style_for_block, list_start_for_block) = if tag == "li" {
2153                        new_list.clone()
2154                    } else {
2155                        (None, None)
2156                    };
2157
2158                    let list_indent_for_block = if tag == "li" {
2159                        new_list_depth.saturating_sub(1)
2160                    } else {
2161                        0
2162                    };
2163
2164                    if !(is_code_block || css.preserve_whitespace == Some(true)) {
2165                        collapse_inline_whitespace(&mut spans);
2166                    }
2167
2168                    // Whitespace between block-level children is layout, not
2169                    // content. `<body>` holds a newline after every `</p>` an
2170                    // exporter writes, and keeping those as the container's own
2171                    // inline run is what closed a pasted book with a paragraph
2172                    // of four thousand blank lines. A block that produced no
2173                    // children keeps its empty run instead: `<p> </p>` is an
2174                    // empty paragraph the writer meant to be there.
2175                    let own_run_is_content =
2176                        spans_carry_content(&spans) || nested_elements.is_empty();
2177
2178                    if (!spans.is_empty() && own_run_is_content) || heading_level.is_some() {
2179                        // A code block is verbatim text, which no save can write an image or
2180                        // a note's reference into: a `<pre>` holding one is read as prose,
2181                        // which keeps it, one paragraph for each of its lines (see below).
2182                        let is_code_block = is_code_block
2183                            && !spans
2184                                .iter()
2185                                .any(|span| span.image.is_some() || span.footnote_ref.is_some());
2186                        let block = ParsedBlock {
2187                            spans,
2188                            heading_level,
2189                            list_style: list_style_for_block,
2190                            list_indent: list_indent_for_block,
2191                            list_prefix: String::new(),
2192                            list_suffix: String::new(),
2193                            list_start: list_start_for_block,
2194                            marker: None,
2195                            is_code_block,
2196                            code_language: code_language.filter(|_| is_code_block),
2197                            blockquote_depth: bq_depth,
2198                            blockquote_opens: 0,
2199                            line_height: css.line_height,
2200                            non_breakable_lines: css.non_breakable_lines,
2201                            page_break_before: css.page_break_before,
2202                            front_matter: None,
2203                            direction: css.direction,
2204                            background_color: css.background_color,
2205                            alignment: None,
2206                            top_margin: None,
2207                            text_indent: None,
2208                            semantic_role: None,
2209                        };
2210                        // Kept whitespace keeps its line breaks, and a line break in
2211                        // prose is a new block to the model. Only a code block holds
2212                        // its lines in one block.
2213                        if is_code_block {
2214                            elements.push(ParsedElement::Block(block));
2215                        } else {
2216                            elements.extend(
2217                                split_block_at_line_breaks(block)
2218                                    .into_iter()
2219                                    .map(ParsedElement::Block),
2220                            );
2221                        }
2222                    }
2223                    // Append nested block elements after the parent block
2224                    elements.append(&mut nested_elements);
2225                } else if matches!(tag, "ul" | "ol" | "thead" | "tbody" | "tr") {
2226                    // Container elements: recurse into children
2227                    for child in node.children() {
2228                        walk_node(
2229                            child,
2230                            &new_state,
2231                            elements,
2232                            &new_list,
2233                            bq_depth,
2234                            new_list_depth,
2235                            depth + 1,
2236                            limit,
2237                        );
2238                    }
2239                } else {
2240                    // Inline element or unknown: recurse
2241                    for child in node.children() {
2242                        walk_node(
2243                            child,
2244                            &new_state,
2245                            elements,
2246                            current_list,
2247                            bq_depth,
2248                            list_depth,
2249                            depth + 1,
2250                            limit,
2251                        );
2252                    }
2253                }
2254            }
2255            Node::Text(text) => {
2256                let t = text.text.to_string();
2257                let trimmed = t.trim();
2258                if !trimmed.is_empty() {
2259                    // Bare text not in a block — create a paragraph
2260                    elements.push(ParsedElement::Block(ParsedBlock {
2261                        spans: vec![ParsedSpan {
2262                            text: trimmed.to_string(),
2263                            bold: state.bold,
2264                            italic: state.italic,
2265                            underline: state.underline,
2266                            strikeout: state.strikeout,
2267                            code: state.code,
2268                            superscript: state.superscript,
2269                            subscript: state.subscript,
2270                            link_href: state.link_href.clone(),
2271                            image: None,
2272                            footnote_ref: None,
2273                        }],
2274                        heading_level: None,
2275                        list_style: None,
2276                        list_indent: 0,
2277                        list_prefix: String::new(),
2278                        list_suffix: String::new(),
2279                        list_start: None,
2280                        marker: None,
2281                        is_code_block: false,
2282                        code_language: None,
2283                        blockquote_depth,
2284                        blockquote_opens: 0,
2285                        line_height: None,
2286                        non_breakable_lines: None,
2287                        page_break_before: None,
2288                        front_matter: None,
2289                        direction: None,
2290                        background_color: None,
2291                        alignment: None,
2292                        top_margin: None,
2293                        text_indent: None,
2294                        semantic_role: None,
2295                    }));
2296                }
2297            }
2298            _ => {
2299                // Document, Comment, etc. — recurse children
2300                for child in node.children() {
2301                    walk_node(
2302                        child,
2303                        state,
2304                        elements,
2305                        current_list,
2306                        blockquote_depth,
2307                        list_depth,
2308                        depth + 1,
2309                        limit,
2310                    );
2311                }
2312            }
2313        }
2314    }
2315
2316    /// Collect inline spans from a block-level element's children.
2317    /// If a nested block-level element is encountered, it is flushed as a
2318    /// separate block.
2319    #[allow(clippy::too_many_arguments)]
2320    fn collect_inline_spans(
2321        node: ego_tree::NodeRef<Node>,
2322        state: &FmtState,
2323        spans: &mut Vec<ParsedSpan>,
2324        current_list: &HtmlList,
2325        elements: &mut Vec<ParsedElement>,
2326        blockquote_depth: u32,
2327        list_depth: u32,
2328        depth: usize,
2329        limit: usize,
2330    ) {
2331        if depth > limit {
2332            let text = subtree_text(node);
2333            if !text.is_empty() {
2334                spans.push(text_span(text, state));
2335            }
2336            return;
2337        }
2338        for child in node.children() {
2339            match child.value() {
2340                Node::Text(text) => {
2341                    let t = text.text.to_string();
2342                    if !t.is_empty() {
2343                        spans.push(ParsedSpan {
2344                            text: t,
2345                            bold: state.bold,
2346                            italic: state.italic,
2347                            underline: state.underline,
2348                            strikeout: state.strikeout,
2349                            code: state.code,
2350                            superscript: state.superscript,
2351                            subscript: state.subscript,
2352                            link_href: state.link_href.clone(),
2353                            image: None,
2354                            footnote_ref: None,
2355                        });
2356                    }
2357                }
2358                Node::Element(el) => {
2359                    let tag = el.name();
2360                    if is_metadata_tag(tag) {
2361                        continue;
2362                    }
2363                    let mut new_state = state.clone();
2364
2365                    match tag {
2366                        // Checked by attribute rather than by tag name: the
2367                        // producer picks the element (a `<sup>` renders sensibly
2368                        // in a browser), and only the attribute is a contract.
2369                        _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
2370                            if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
2371                            {
2372                                spans.push(span);
2373                            }
2374                            continue;
2375                        }
2376                        "b" | "strong" => new_state.bold = true,
2377                        "i" | "em" => new_state.italic = true,
2378                        "u" | "ins" => new_state.underline = true,
2379                        "s" | "del" | "strike" => new_state.strikeout = true,
2380                        "code" => new_state.code = true,
2381                        "sup" => new_state.superscript = true,
2382                        "sub" => new_state.subscript = true,
2383                        "a" => {
2384                            if let Some(href) = el.attr("href") {
2385                                new_state.link_href = Some(href.to_string());
2386                            }
2387                        }
2388                        "img" => {
2389                            if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
2390                                spans.push(span);
2391                            }
2392                            continue;
2393                        }
2394                        _ => {}
2395                    }
2396
2397                    // Check for nested block elements
2398                    let nested_block = matches!(
2399                        tag,
2400                        "p" | "div"
2401                            | "h1"
2402                            | "h2"
2403                            | "h3"
2404                            | "h4"
2405                            | "h5"
2406                            | "h6"
2407                            | "li"
2408                            | "pre"
2409                            | "blockquote"
2410                            | "ul"
2411                            | "ol"
2412                    );
2413
2414                    if tag == "br" {
2415                        // br within a block: treat as splitting into new block
2416                        // For simplicity, just add a newline to current span
2417                        spans.push(ParsedSpan {
2418                            text: String::new(),
2419                            ..Default::default()
2420                        });
2421                    } else if nested_block || tag == "table" {
2422                        // Flush as separate element
2423                        walk_node(
2424                            child,
2425                            &new_state,
2426                            elements,
2427                            current_list,
2428                            blockquote_depth,
2429                            list_depth,
2430                            depth + 1,
2431                            limit,
2432                        );
2433                    } else {
2434                        // Inline element: recurse
2435                        collect_inline_spans(
2436                            child,
2437                            &new_state,
2438                            spans,
2439                            current_list,
2440                            elements,
2441                            blockquote_depth,
2442                            list_depth,
2443                            depth + 1,
2444                            limit,
2445                        );
2446                    }
2447                }
2448                _ => {}
2449            }
2450        }
2451    }
2452
2453    let initial_state = FmtState::default();
2454    // Treat the root element as a block-level container so that
2455    // top-level inline elements (e.g. `<b>Bold</b> <em>Italic</em>`)
2456    // are grouped into a single block instead of becoming separate blocks.
2457    let mut root_spans: Vec<ParsedSpan> = Vec::new();
2458    collect_inline_spans(
2459        *root,
2460        &initial_state,
2461        &mut root_spans,
2462        &(None, None),
2463        &mut elements,
2464        0,
2465        0,
2466        0,
2467        depth_limit,
2468    );
2469    collapse_inline_whitespace(&mut root_spans);
2470    // Only when something survived: the root's direct text children are the
2471    // newlines between top-level blocks in every document an exporter writes.
2472    if spans_carry_content(&root_spans) {
2473        elements.push(ParsedElement::Block(ParsedBlock {
2474            spans: root_spans,
2475            heading_level: None,
2476            list_style: None,
2477            list_indent: 0,
2478            list_prefix: String::new(),
2479            list_suffix: String::new(),
2480            list_start: None,
2481            marker: None,
2482            is_code_block: false,
2483            code_language: None,
2484            blockquote_depth: 0,
2485            blockquote_opens: 0,
2486            line_height: None,
2487            non_breakable_lines: None,
2488            page_break_before: None,
2489            front_matter: None,
2490            direction: None,
2491            background_color: None,
2492            alignment: None,
2493            top_margin: None,
2494            text_indent: None,
2495            semantic_role: None,
2496        }));
2497    }
2498
2499    // If no elements were parsed, create a single empty paragraph
2500    if elements.is_empty() {
2501        elements.push(ParsedElement::Block(ParsedBlock {
2502            spans: vec![ParsedSpan {
2503                text: String::new(),
2504                ..Default::default()
2505            }],
2506            heading_level: None,
2507            list_style: None,
2508            list_indent: 0,
2509            list_prefix: String::new(),
2510            list_suffix: String::new(),
2511            list_start: None,
2512            marker: None,
2513            is_code_block: false,
2514            code_language: None,
2515            blockquote_depth: 0,
2516            blockquote_opens: 0,
2517            line_height: None,
2518            non_breakable_lines: None,
2519            page_break_before: None,
2520            front_matter: None,
2521            direction: None,
2522            background_color: None,
2523            alignment: None,
2524            top_margin: None,
2525            text_indent: None,
2526            semantic_role: None,
2527        }));
2528    }
2529
2530    elements
2531}
2532
2533/// Convert a `ParsedSpan` (parser output) into the `CharacterFormat` used by
2534/// `FormatRun`. `is_code_block` forces `monospace` as the font family for
2535/// every span inside a code block.
2536pub fn character_format_from_span(
2537    span: &ParsedSpan,
2538    is_code_block: bool,
2539) -> crate::format_runs::CharacterFormat {
2540    use crate::entities::CharVerticalAlignment;
2541    crate::format_runs::CharacterFormat {
2542        font_bold: if span.bold { Some(true) } else { None },
2543        font_italic: if span.italic { Some(true) } else { None },
2544        font_underline: if span.underline { Some(true) } else { None },
2545        font_strikeout: if span.strikeout { Some(true) } else { None },
2546        font_family: if span.code || is_code_block {
2547            Some("monospace".to_string())
2548        } else {
2549            None
2550        },
2551        anchor_href: span.link_href.clone(),
2552        is_anchor: if span.link_href.is_some() {
2553            Some(true)
2554        } else {
2555            None
2556        },
2557        vertical_alignment: if span.superscript {
2558            Some(CharVerticalAlignment::SuperScript)
2559        } else if span.subscript {
2560            Some(CharVerticalAlignment::SubScript)
2561        } else {
2562            None
2563        },
2564        ..Default::default()
2565    }
2566}
2567
2568/// Translate a slice of parsed spans into `(plain_text, format_runs)`.
2569///
2570/// One non-default span yields one `FormatRun`; spans with empty
2571/// `CharacterFormat` (no decoration, no link, no code) emit no run, since an
2572/// absent run means "inherit default formatting" in the new model. Adjacent
2573/// runs with identical formats are coalesced via `coalesce_in_place` so the
2574/// resulting vector satisfies `debug_assert_well_formed`.
2575///
2576/// Returns the concatenated `plain_text` of all spans and a sorted,
2577/// non-overlapping, coalesced `Vec<FormatRun>`. Both safe to feed straight
2578/// into the store under the dual-write bridge.
2579pub fn format_runs_from_spans(spans: &[ParsedSpan], is_code_block: bool) -> ParsedInline {
2580    use crate::format_runs::{
2581        CharacterFormat, FootnoteRefAnchor, FormatRun, ImageAnchor, coalesce_in_place,
2582    };
2583
2584    let mut plain_text = String::new();
2585    let mut runs: Vec<FormatRun> = Vec::new();
2586    let mut images: Vec<ImageAnchor> = Vec::new();
2587    let mut footnote_refs: Vec<FootnoteRefAnchor> = Vec::new();
2588    let default = CharacterFormat::default();
2589
2590    for span in spans {
2591        let byte_start = plain_text.len() as u32;
2592
2593        if let Some(label) = &span.footnote_ref {
2594            // A reference occupies one U+FFFC, exactly as an image does, so
2595            // every downstream offset treats the two alike.
2596            plain_text.push('\u{FFFC}');
2597            // Raised, always — whatever the surrounding run is doing.
2598            //
2599            // A footnote marker is superscript by definition, in every
2600            // typographic tradition and in every reader that renders djot. The
2601            // ambient `superscript` flag the span carries is the *prose's*, and
2602            // prose is not superscript, so taking it verbatim sets a note's
2603            // number on the baseline in the middle of a sentence — which reads
2604            // as a stray digit the writer typed rather than as a reference.
2605            //
2606            // It is set here, on the anchor, rather than at render time so that
2607            // every consumer agrees: the editor raises it, the exporters that
2608            // carry character formatting carry it, and the djot writer knows to
2609            // emit `[^label]` *without* wrapping it in `^…^` (see
2610            // `a_reference_is_not_wrapped_in_superscript_markers`).
2611            let mut format = character_format_from_span(span, is_code_block);
2612            format.vertical_alignment = Some(crate::entities::CharVerticalAlignment::SuperScript);
2613            footnote_refs.push(FootnoteRefAnchor {
2614                byte_offset: byte_start,
2615                label: label.clone(),
2616                format,
2617            });
2618            continue;
2619        }
2620
2621        if let Some(image) = &span.image {
2622            // An image occupies one U+FFFC in the text, exactly as
2623            // `insert_image` mirrors into the rope, so every downstream offset
2624            // calculation treats a parsed image and an inserted one alike.
2625            plain_text.push('\u{FFFC}');
2626            images.push(ImageAnchor {
2627                byte_offset: byte_start,
2628                name: image.src.clone(),
2629                alt: image.alt.clone(),
2630                width: image.width,
2631                height: image.height,
2632                quality: 100,
2633                format: character_format_from_span(span, is_code_block),
2634            });
2635            continue;
2636        }
2637
2638        plain_text.push_str(&span.text);
2639        let byte_end = plain_text.len() as u32;
2640        if byte_start == byte_end {
2641            continue;
2642        }
2643        let format = character_format_from_span(span, is_code_block);
2644        if format == default {
2645            continue;
2646        }
2647        runs.push(FormatRun {
2648            byte_start,
2649            byte_end,
2650            format,
2651        });
2652    }
2653    coalesce_in_place(&mut runs);
2654    ParsedInline {
2655        plain_text,
2656        runs,
2657        images,
2658        footnote_refs,
2659    }
2660}
2661
2662/// The three parallel things a block stores, as recovered from parsed spans.
2663///
2664/// Returned as a struct rather than a tuple because it grew a third member
2665/// (images) after nine call sites already destructured a pair — and every one
2666/// of those sites has to decide what to do with images, so a silent
2667/// tuple-arity change would have been the wrong kind of easy.
2668#[derive(Debug, Clone, Default)]
2669pub struct ParsedInline {
2670    pub plain_text: String,
2671    pub runs: Vec<crate::format_runs::FormatRun>,
2672    pub images: Vec<crate::format_runs::ImageAnchor>,
2673    pub footnote_refs: Vec<crate::format_runs::FootnoteRefAnchor>,
2674}
2675
2676// ─── Djot parsing ────────────────────────────────────────────────────
2677
2678/// Map a jotdown unordered/task bullet marker to a model `ListStyle`.
2679///
2680/// The mapping is a stable bijection (`-`↔Disc, `*`↔Circle, `+`↔Square) so the
2681/// djot exporter can recover the exact bullet character for a lossless
2682/// round-trip.
2683fn djot_bullet_style(b: jotdown::ListBulletType) -> ListStyle {
2684    use jotdown::ListBulletType as B;
2685    match b {
2686        B::Dash => ListStyle::Disc,
2687        B::Star => ListStyle::Circle,
2688        B::Plus => ListStyle::Square,
2689    }
2690}
2691
2692/// Map a jotdown ordered-list numbering scheme to a model `ListStyle`.
2693fn djot_ordered_style(n: jotdown::OrderedListNumbering) -> ListStyle {
2694    use jotdown::OrderedListNumbering as N;
2695    match n {
2696        N::Decimal => ListStyle::Decimal,
2697        N::AlphaLower => ListStyle::LowerAlpha,
2698        N::AlphaUpper => ListStyle::UpperAlpha,
2699        N::RomanLower => ListStyle::LowerRoman,
2700        N::RomanUpper => ListStyle::UpperRoman,
2701    }
2702}
2703
2704/// Map a jotdown ordered-list delimiter to the `(prefix, suffix)` affixes
2705/// stored on the `List` entity (`1.` → `("", ".")`, `1)` → `("", ")")`,
2706/// `(1)` → `("(", ")")`).
2707fn djot_ordered_affixes(style: jotdown::OrderedListStyle) -> (String, String) {
2708    use jotdown::OrderedListStyle as S;
2709    match style {
2710        S::Period => (String::new(), ".".to_string()),
2711        S::Paren => (String::new(), ")".to_string()),
2712        S::ParenParen => ("(".to_string(), ")".to_string()),
2713    }
2714}
2715
2716/// Optional block-level style attributes carried on a djot block through its
2717/// `{key=value}` block attributes. All `None` when the block has no such
2718/// attributes (or they were filtered out by [`DjotImportOptions`]).
2719#[derive(Debug, Clone, Default)]
2720struct DjotBlockStyle {
2721    alignment: Option<Alignment>,
2722    line_height: Option<i64>,
2723    non_breakable_lines: Option<bool>,
2724    page_break_before: Option<bool>,
2725    front_matter: Option<bool>,
2726    direction: Option<TextDirection>,
2727    background_color: Option<String>,
2728    top_margin: Option<i64>,
2729    text_indent: Option<i64>,
2730    semantic_role: Option<SemanticRole>,
2731}
2732
2733impl DjotBlockStyle {
2734    /// Overlay the `Some` fields of `other` onto `self`, leaving `self`'s
2735    /// existing values for any field `other` does not set. Used to combine a
2736    /// heading's enclosing-`Section` attributes with any on the heading itself.
2737    fn merge_from(&mut self, other: DjotBlockStyle) {
2738        if other.alignment.is_some() {
2739            self.alignment = other.alignment;
2740        }
2741        if other.line_height.is_some() {
2742            self.line_height = other.line_height;
2743        }
2744        if other.non_breakable_lines.is_some() {
2745            self.non_breakable_lines = other.non_breakable_lines;
2746        }
2747        if other.page_break_before.is_some() {
2748            self.page_break_before = other.page_break_before;
2749        }
2750        if other.front_matter.is_some() {
2751            self.front_matter = other.front_matter;
2752        }
2753        if other.direction.is_some() {
2754            self.direction = other.direction;
2755        }
2756        if other.background_color.is_some() {
2757            self.background_color = other.background_color;
2758        }
2759        if other.top_margin.is_some() {
2760            self.top_margin = other.top_margin;
2761        }
2762        if other.text_indent.is_some() {
2763            self.text_indent = other.text_indent;
2764        }
2765        if other.semantic_role.is_some() {
2766            self.semantic_role = other.semantic_role.clone();
2767        }
2768    }
2769}
2770
2771/// Read the round-tripped block-style attributes off a djot block's
2772/// [`jotdown::Attributes`], honouring the import [`DjotImportOptions`]. Keys are
2773/// the model field names (`alignment`, `line_height`, `direction`,
2774/// `non_breakable_lines`, `page_break_before`, `front_matter`, `background_color`,
2775/// `top_margin`, `text_indent`, `semantic_role`); unrecognised values are ignored.
2776fn block_attrs_to_style(attrs: &jotdown::Attributes, opts: &DjotImportOptions) -> DjotBlockStyle {
2777    let mut style = DjotBlockStyle::default();
2778
2779    if opts.alignment
2780        && let Some(v) = attrs.get_value("alignment")
2781    {
2782        style.alignment = match v.to_string().as_str() {
2783            "left" => Some(Alignment::Left),
2784            "right" => Some(Alignment::Right),
2785            "center" => Some(Alignment::Center),
2786            "justify" => Some(Alignment::Justify),
2787            _ => None,
2788        };
2789    }
2790    if opts.line_height
2791        && let Some(v) = attrs.get_value("line_height")
2792    {
2793        style.line_height = v.to_string().parse::<i64>().ok();
2794    }
2795    if opts.direction
2796        && let Some(v) = attrs.get_value("direction")
2797    {
2798        style.direction = match v.to_string().as_str() {
2799            "ltr" => Some(TextDirection::LeftToRight),
2800            "rtl" => Some(TextDirection::RightToLeft),
2801            _ => None,
2802        };
2803    }
2804    if opts.non_breakable_lines
2805        && let Some(v) = attrs.get_value("non_breakable_lines")
2806    {
2807        style.non_breakable_lines = match v.to_string().as_str() {
2808            "true" => Some(true),
2809            "false" => Some(false),
2810            _ => None,
2811        };
2812    }
2813    if opts.page_break_before
2814        && let Some(v) = attrs.get_value("page_break_before")
2815    {
2816        style.page_break_before = match v.to_string().as_str() {
2817            "true" => Some(true),
2818            "false" => Some(false),
2819            _ => None,
2820        };
2821    }
2822    if opts.front_matter
2823        && let Some(v) = attrs.get_value("front_matter")
2824    {
2825        style.front_matter = match v.to_string().as_str() {
2826            "true" => Some(true),
2827            "false" => Some(false),
2828            _ => None,
2829        };
2830    }
2831    if opts.background_color
2832        && let Some(v) = attrs.get_value("background_color")
2833    {
2834        style.background_color = Some(v.to_string());
2835    }
2836    if opts.top_margin
2837        && let Some(v) = attrs.get_value("top_margin")
2838    {
2839        style.top_margin = v.to_string().parse::<i64>().ok();
2840    }
2841    if opts.text_indent
2842        && let Some(v) = attrs.get_value("text_indent")
2843    {
2844        style.text_indent = v.to_string().parse::<i64>().ok();
2845    }
2846    if opts.semantic_role
2847        && let Some(v) = attrs.get_value("semantic_role")
2848    {
2849        style.semantic_role = match v.to_string().as_str() {
2850            "epigraph" => Some(SemanticRole::Epigraph),
2851            // An unknown role is dropped, not guessed at — the same way an unknown
2852            // alignment value above is. A future role read by an older build then
2853            // degrades to a plain blockquote, which is what it looks like anyway.
2854            _ => None,
2855        };
2856    }
2857
2858    style
2859}
2860
2861/// The start a list keeps on its entity, from the number its first item is written with:
2862/// `None` for 1, which every list starts at unless it says otherwise, and a number past what
2863/// every format holds read as the largest it holds (see `list_start`).
2864fn list_start_of(start: u64) -> Option<i64> {
2865    crate::parser_tools::list_start::list_start_from_unsigned(start)
2866}
2867
2868/// Push a finished block into `elements`, applying the djot block-level fields
2869/// plus any round-tripped block-style attributes carried in `style`.
2870#[allow(clippy::too_many_arguments)]
2871fn djot_push_block(
2872    elements: &mut Vec<ParsedElement>,
2873    spans: Vec<ParsedSpan>,
2874    heading_level: Option<i64>,
2875    list_style: Option<ListStyle>,
2876    list_indent: u32,
2877    list_prefix: String,
2878    list_suffix: String,
2879    list_start: Option<i64>,
2880    marker: Option<MarkerType>,
2881    is_code_block: bool,
2882    code_language: Option<String>,
2883    blockquote_depth: u32,
2884    style: DjotBlockStyle,
2885) {
2886    elements.push(ParsedElement::Block(ParsedBlock {
2887        spans,
2888        heading_level,
2889        list_style,
2890        list_indent,
2891        list_prefix,
2892        list_suffix,
2893        list_start,
2894        marker,
2895        is_code_block,
2896        code_language,
2897        blockquote_depth,
2898        blockquote_opens: 0,
2899        line_height: style.line_height,
2900        non_breakable_lines: style.non_breakable_lines,
2901        page_break_before: style.page_break_before,
2902        front_matter: style.front_matter,
2903        direction: style.direction,
2904        background_color: style.background_color,
2905        alignment: style.alignment,
2906        top_margin: style.top_margin,
2907        text_indent: style.text_indent,
2908        semantic_role: style.semantic_role.clone(),
2909    }));
2910}
2911
2912/// A Djot document whose structure cannot be computed, as its source: one plain
2913/// paragraph for each line that holds text, with that line's text as it is written, its
2914/// leading and trailing whitespace aside.
2915///
2916/// A paragraph a line is what keeps the text the same through a save. The Djot writer
2917/// escapes whatever a paragraph opens with and whatever in it would read as markup, so
2918/// the saved text reads back as these same paragraphs. Shown as one paragraph, the
2919/// source kept its line breaks through a save, read back as the same document the parser
2920/// could not take, and every save escaped its backslashes once more. Blank lines are left
2921/// out because the reader keeps no block for an empty paragraph, and a line's outer
2922/// whitespace because it is the source's indentation, which the parser does not read as
2923/// text in a line it can take either.
2924fn djot_source_lines(djot: &str) -> Vec<ParsedElement> {
2925    djot.lines()
2926        .map(|line| line.trim_matches(|c: char| c.is_ascii_whitespace()))
2927        .filter(|line| !line.is_empty())
2928        .map(|line| {
2929            ParsedElement::Block(ParsedBlock {
2930                spans: vec![ParsedSpan {
2931                    text: line.to_string(),
2932                    ..Default::default()
2933                }],
2934                ..Default::default()
2935            })
2936        })
2937        .collect()
2938}
2939
2940/// Parse djot source into the shared [`ParsedElement`] intermediate, mirroring
2941/// [`parse_markdown`]. Uses the [`jotdown`] pull parser.
2942///
2943/// Constructs the document model cannot represent are dropped, and their text
2944/// content is discarded so it never leaks into the document: footnotes, math,
2945/// fenced divs, raw blocks/inline, thematic breaks, description lists,
2946/// captions, symbols, link-reference definitions, and highlight/`mark`. Inline
2947/// images keep their alt text as plain text (the image itself is not modelled),
2948/// matching the Markdown importer. Smart-punctuation events are normalised to
2949/// their canonical Unicode characters so the model→djot→model round-trip is a
2950/// fixpoint.
2951///
2952/// Standalone paragraphs and headings additionally carry the optional
2953/// block-style attributes selected by `options` — paragraph alignment, line
2954/// height, text direction, non-breakable lines and background color — read from
2955/// djot `{key=value}` block attributes (see [`DjotImportOptions`]). List items,
2956/// code blocks and table cells normalise their block styling away.
2957///
2958/// Known model limitations (normalised, not preserved on round-trip):
2959/// table column alignment, and list tight/loose.
2960pub fn parse_djot(djot: &str, options: &DjotImportOptions) -> Vec<ParsedElement> {
2961    use jotdown::{Container as C, Event as E, ListKind};
2962
2963    // `jotdown` recurses with no limit of its own, once per nested container and
2964    // once per line of a paragraph that leaves an inline opener waiting, and a
2965    // stack overflow **aborts the process**: it cannot be caught by
2966    // `catch_unwind`, so there is no recovering from it after the fact and the
2967    // only safe move is not to start. It also panics on a heading deeper than it
2968    // can number, which a caller on an interface thread does not survive either.
2969    // Input is not always the author's own: a project bundle is mailed and
2970    // shared, and an imported document comes from whoever sent it.
2971    //
2972    // `djot_depth::parsable` is the door that keeps the parser within its limits,
2973    // and it degrades rather than refuses, so the signature stays the same and no
2974    // word is lost. Such a heading is escaped into text, as it was written. A
2975    // document nested too deeply by its indentation, which is how a deeply nested
2976    // list is written, is read with the indentation cut: the list comes out
2977    // flattened below a fixed depth, every item and word kept. This used to hand
2978    // back a pasted list nested two hundred levels as one paragraph of markup. A
2979    // long paragraph is parsed on a stack sized for it, and the block holding one
2980    // too long for any stack is shown as its lines, the rest of the document as it
2981    // was.
2982    //
2983    // A document still nested too deeply once flattened, or one the parser panicked
2984    // on, comes back as its lines, one plain paragraph each (see
2985    // `djot_source_lines`). Its structure is not computed, which is the honest
2986    // answer for a document whose structure cannot be computed without ending the
2987    // process, and its text is still there for the writer to see and repair.
2988    let parsable = crate::parser_tools::djot_depth::parsable(djot);
2989    let Some(events) = parsable.as_ref().and_then(|parsable| parsable.events()) else {
2990        return djot_source_lines(djot);
2991    };
2992
2993    let mut elements: Vec<ParsedElement> = Vec::new();
2994    let mut current_spans: Vec<ParsedSpan> = Vec::new();
2995    let mut current_heading: Option<i64> = None;
2996    let mut is_code_block = false;
2997    let mut code_language: Option<String> = None;
2998    let mut blockquote_depth: u32 = 0;
2999    // Where `elements` stood as each open quotation began (see `mark_quotation_opened`).
3000    let mut quotations_open: Vec<usize> = Vec::new();
3001    // Block-style attributes captured from a standalone paragraph/heading's djot
3002    // `{…}` block attributes, consumed when that block is flushed.
3003    let mut pending_style = DjotBlockStyle::default();
3004
3005    // Inline formatting state.
3006    let mut bold = false;
3007    let mut italic = false;
3008    let mut underline = false;
3009    let mut strikeout = false;
3010    let mut code = false;
3011    let mut superscript = false;
3012    let mut subscript = false;
3013    let mut link_href: Option<String> = None;
3014    // Set between an image's Start and End. Its alt text arrives as ordinary
3015    // `Str` events in between, so it has to be captured rather than emitted.
3016    let mut pending_image: Option<ParsedImage> = None;
3017
3018    // List nesting: each entry is (style, prefix, suffix, start); depth = indent + 1.
3019    let mut list_stack: Vec<(ListStyle, String, String, Option<i64>)> = Vec::new();
3020    // Context applied to the next flushed block while inside a list item.
3021    let mut cur_list_style: Option<ListStyle> = None;
3022    let mut cur_list_prefix = String::new();
3023    let mut cur_list_suffix = String::new();
3024    let mut cur_list_start: Option<i64> = None;
3025    let mut cur_list_indent: u32 = 0;
3026    let mut cur_marker: Option<MarkerType> = None;
3027
3028    // Table accumulation.
3029    let mut in_table_cell = false;
3030    let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
3031    let mut current_row: Vec<ParsedTableCell> = Vec::new();
3032    let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
3033    let mut table_header_rows: usize = 0;
3034    let mut row_is_head = false;
3035
3036    // Subtree-skip depth for unrepresentable containers (their entire content
3037    // is dropped). Incremented on the dropped container's `Start` and on every
3038    // nested `Start`; decremented on every `End`.
3039    let mut skip_depth: u32 = 0;
3040
3041    // The label and element index of the footnote definition currently open,
3042    // if any. Djot has no footnote inside a footnote, so one slot is enough
3043    // where the dropped containers need a depth counter.
3044    let mut footnote_open: Option<(String, usize)> = None;
3045
3046    // Push one inline span carrying the current formatting state into the
3047    // active sink (table cell or block). A macro (not a closure) to avoid
3048    // borrowing `current_spans`/`current_cell_spans` across the formatting
3049    // state reads.
3050    macro_rules! push_text {
3051        ($t:expr) => {{
3052            // Alt text belongs to the image, not to the paragraph. While an
3053            // image is open every text event is diverted into its description,
3054            // which is what keeps a photo's caption out of the manuscript's
3055            // word count and out of the search corpus.
3056            if let Some(img) = pending_image.as_mut() {
3057                img.alt.push_str(($t).as_ref());
3058            } else {
3059                let sp = ParsedSpan {
3060                    text: ($t).to_string(),
3061                    bold,
3062                    italic,
3063                    underline,
3064                    strikeout,
3065                    code,
3066                    superscript,
3067                    subscript,
3068                    link_href: link_href.clone(),
3069                    image: None,
3070                    footnote_ref: None,
3071                };
3072                if in_table_cell {
3073                    current_cell_spans.push(sp);
3074                } else {
3075                    current_spans.push(sp);
3076                }
3077            }
3078        }};
3079    }
3080
3081    // Push a completed inline image span into the active sink.
3082    macro_rules! push_image {
3083        ($img:expr) => {{
3084            let sp = ParsedSpan {
3085                text: String::new(),
3086                bold,
3087                italic,
3088                underline,
3089                strikeout,
3090                code,
3091                superscript,
3092                subscript,
3093                link_href: link_href.clone(),
3094                image: Some($img),
3095                footnote_ref: None,
3096            };
3097            if in_table_cell {
3098                current_cell_spans.push(sp);
3099            } else {
3100                current_spans.push(sp);
3101            }
3102        }};
3103    }
3104
3105    // Enter a list item, flushing any unterminated inline content first and
3106    // capturing the list context + task marker for the item's block.
3107    macro_rules! enter_item {
3108        ($marker:expr) => {{
3109            if !current_spans.is_empty() {
3110                djot_push_block(
3111                    &mut elements,
3112                    std::mem::take(&mut current_spans),
3113                    None,
3114                    cur_list_style.clone(),
3115                    cur_list_indent,
3116                    cur_list_prefix.clone(),
3117                    cur_list_suffix.clone(),
3118                    cur_list_start,
3119                    cur_marker.clone(),
3120                    false,
3121                    None,
3122                    blockquote_depth,
3123                    DjotBlockStyle::default(),
3124                );
3125            }
3126            let (style, prefix, suffix, start) = list_stack.last().cloned().unwrap_or((
3127                ListStyle::Disc,
3128                String::new(),
3129                String::new(),
3130                None,
3131            ));
3132            cur_list_style = Some(style);
3133            cur_list_prefix = prefix;
3134            cur_list_suffix = suffix;
3135            cur_list_start = start;
3136            cur_list_indent = list_stack.len().saturating_sub(1) as u32;
3137            cur_marker = $marker;
3138        }};
3139    }
3140
3141    // A line break inside a block: a new block, which is what the model means by one,
3142    // with the list, quotation, heading and block style the line was in. A table cell
3143    // is one line, so there it is a space.
3144    //
3145    // The page break is the one style that stays with the first line: it is where the
3146    // block starts, and copied to every line it put each on a page of its own.
3147    macro_rules! line_break {
3148        () => {{
3149            if in_table_cell {
3150                push_text!(" ");
3151            } else if !current_spans.is_empty() && current_heading.is_some() {
3152                // The line before a hard break in a heading is a line of that
3153                // heading, as the line after it is when the heading ends. Read as
3154                // a paragraph, a heading written over two lines came back as a
3155                // paragraph and a heading.
3156                djot_push_block(
3157                    &mut elements,
3158                    std::mem::take(&mut current_spans),
3159                    current_heading,
3160                    None,
3161                    0,
3162                    String::new(),
3163                    String::new(),
3164                    None,
3165                    None,
3166                    false,
3167                    None,
3168                    blockquote_depth,
3169                    pending_style.clone(),
3170                );
3171                pending_style.page_break_before = None;
3172            } else if !current_spans.is_empty() {
3173                // Mirrors the Markdown importer: a hard break splits the
3174                // paragraph into a new block.
3175                djot_push_block(
3176                    &mut elements,
3177                    std::mem::take(&mut current_spans),
3178                    None,
3179                    cur_list_style.clone(),
3180                    cur_list_indent,
3181                    cur_list_prefix.clone(),
3182                    cur_list_suffix.clone(),
3183                    cur_list_start,
3184                    cur_marker.clone(),
3185                    is_code_block,
3186                    code_language.clone(),
3187                    blockquote_depth,
3188                    pending_style.clone(),
3189                );
3190                pending_style.page_break_before = None;
3191            }
3192        }};
3193    }
3194
3195    for event in events {
3196        if skip_depth > 0 {
3197            match event {
3198                E::Start(..) => skip_depth += 1,
3199                E::End(_) => skip_depth -= 1,
3200                _ => {}
3201            }
3202            continue;
3203        }
3204
3205        match event {
3206            // ── Transparent wrappers (unwrap, keep content) ──
3207            E::Start(C::Document, _) | E::End(C::Document) => {}
3208            E::Start(C::Section { .. }, attrs) => {
3209                // A heading's block attributes attach to its enclosing Section,
3210                // not the heading itself; capture them for the heading's flush.
3211                if list_stack.is_empty() {
3212                    pending_style.merge_from(block_attrs_to_style(&attrs, options));
3213                }
3214            }
3215            E::End(C::Section { .. }) => {}
3216            E::Start(C::Div { .. }, _) | E::End(C::Div { .. }) => {}
3217
3218            // ── Blockquote ──
3219            E::Start(C::Blockquote, _) => {
3220                blockquote_depth += 1;
3221                quotations_open.push(elements.len());
3222            }
3223            E::End(C::Blockquote) => {
3224                blockquote_depth = blockquote_depth.saturating_sub(1);
3225                if let Some(first) = quotations_open.pop() {
3226                    mark_quotation_opened(&mut elements, first);
3227                }
3228            }
3229
3230            // ── Lists ──
3231            E::Start(C::List { kind, .. }, _) => {
3232                let (style, prefix, suffix, start) = match kind {
3233                    ListKind::Unordered(b) | ListKind::Task(b) => {
3234                        (djot_bullet_style(b), String::new(), String::new(), None)
3235                    }
3236                    ListKind::Ordered {
3237                        numbering,
3238                        style,
3239                        start,
3240                    } => {
3241                        let (p, s) = djot_ordered_affixes(style);
3242                        (djot_ordered_style(numbering), p, s, list_start_of(start))
3243                    }
3244                };
3245                list_stack.push((style, prefix, suffix, start));
3246            }
3247            E::End(C::List { .. }) => {
3248                list_stack.pop();
3249                cur_list_style = None;
3250                cur_list_start = None;
3251                cur_marker = None;
3252            }
3253            E::Start(C::ListItem, _) => enter_item!(None),
3254            E::Start(C::TaskListItem { checked }, _) => enter_item!(Some(if checked {
3255                MarkerType::Checked
3256            } else {
3257                MarkerType::Unchecked
3258            })),
3259            E::End(C::ListItem) | E::End(C::TaskListItem { .. }) => {
3260                // Tight item without a wrapping paragraph (defensive flush).
3261                if !current_spans.is_empty() {
3262                    djot_push_block(
3263                        &mut elements,
3264                        std::mem::take(&mut current_spans),
3265                        None,
3266                        cur_list_style.clone(),
3267                        cur_list_indent,
3268                        cur_list_prefix.clone(),
3269                        cur_list_suffix.clone(),
3270                        cur_list_start,
3271                        cur_marker.clone(),
3272                        false,
3273                        None,
3274                        blockquote_depth,
3275                        DjotBlockStyle::default(),
3276                    );
3277                }
3278                cur_list_style = None;
3279                cur_list_start = None;
3280                cur_marker = None;
3281            }
3282
3283            // ── Headings, paragraphs, code blocks ──
3284            E::Start(C::Heading { level, .. }, attrs) => {
3285                current_heading = Some(level as i64);
3286                // The block-style attributes live on the enclosing Section;
3287                // merge any placed directly on the heading without clearing them.
3288                pending_style.merge_from(block_attrs_to_style(&attrs, options));
3289            }
3290            E::End(C::Heading { .. }) => {
3291                djot_push_block(
3292                    &mut elements,
3293                    std::mem::take(&mut current_spans),
3294                    current_heading.take(),
3295                    None,
3296                    0,
3297                    String::new(),
3298                    String::new(),
3299                    None,
3300                    None,
3301                    false,
3302                    None,
3303                    blockquote_depth,
3304                    std::mem::take(&mut pending_style),
3305                );
3306            }
3307            E::Start(C::Paragraph, attrs) => {
3308                current_heading = None;
3309                // Block attributes only apply to standalone paragraphs;
3310                // list-item paragraphs normalise their styling away (matching
3311                // the exporter).
3312                pending_style = if list_stack.is_empty() {
3313                    block_attrs_to_style(&attrs, options)
3314                } else {
3315                    DjotBlockStyle::default()
3316                };
3317            }
3318            E::End(C::Paragraph) => {
3319                if !current_spans.is_empty() {
3320                    djot_push_block(
3321                        &mut elements,
3322                        std::mem::take(&mut current_spans),
3323                        None,
3324                        cur_list_style.clone(),
3325                        cur_list_indent,
3326                        cur_list_prefix.clone(),
3327                        cur_list_suffix.clone(),
3328                        cur_list_start,
3329                        cur_marker.clone(),
3330                        false,
3331                        None,
3332                        blockquote_depth,
3333                        std::mem::take(&mut pending_style),
3334                    );
3335                }
3336                cur_list_style = None;
3337                cur_list_start = None;
3338                cur_marker = None;
3339            }
3340            E::Start(C::CodeBlock { language }, _) => {
3341                is_code_block = true;
3342                code_language = if language.is_empty() {
3343                    None
3344                } else {
3345                    Some(language.to_string())
3346                };
3347            }
3348            E::End(C::CodeBlock { .. }) => {
3349                // Strip the single trailing newline jotdown appends.
3350                if let Some(last) = current_spans.last_mut()
3351                    && last.text.ends_with('\n')
3352                {
3353                    last.text.pop();
3354                }
3355                djot_push_block(
3356                    &mut elements,
3357                    std::mem::take(&mut current_spans),
3358                    None,
3359                    None,
3360                    0,
3361                    String::new(),
3362                    String::new(),
3363                    None,
3364                    None,
3365                    true,
3366                    code_language.take(),
3367                    blockquote_depth,
3368                    DjotBlockStyle::default(),
3369                );
3370                is_code_block = false;
3371            }
3372
3373            // ── Tables ──
3374            E::Start(C::Table, _) => {
3375                table_rows.clear();
3376                current_row.clear();
3377                current_cell_spans.clear();
3378                table_header_rows = 0;
3379            }
3380            E::End(C::Table) => {
3381                let own_cells = cells_in_rows(&table_rows);
3382                push_table(
3383                    &mut elements,
3384                    ParsedTable {
3385                        header_rows: table_header_rows,
3386                        rows: std::mem::take(&mut table_rows),
3387                        blockquote_depth,
3388                        blockquote_opens: 0,
3389                    },
3390                    own_cells,
3391                );
3392            }
3393            E::Start(C::TableRow { head }, _) => {
3394                row_is_head = head;
3395                current_row.clear();
3396            }
3397            E::End(C::TableRow { .. }) => {
3398                if row_is_head {
3399                    table_header_rows += 1;
3400                }
3401                table_rows.push(std::mem::take(&mut current_row));
3402            }
3403            E::Start(C::TableCell { .. }, _) => {
3404                in_table_cell = true;
3405                current_cell_spans.clear();
3406            }
3407            E::End(C::TableCell { .. }) => {
3408                in_table_cell = false;
3409                current_row.push(ParsedTableCell {
3410                    spans: std::mem::take(&mut current_cell_spans),
3411                });
3412            }
3413
3414            // ── Inline formatting ──
3415            E::Start(C::Strong, _) => bold = true,
3416            E::End(C::Strong) => bold = false,
3417            E::Start(C::Emphasis, _) => italic = true,
3418            E::End(C::Emphasis) => italic = false,
3419            E::Start(C::Verbatim, _) => code = true,
3420            E::End(C::Verbatim) => code = false,
3421            E::Start(C::Superscript, _) => superscript = true,
3422            E::End(C::Superscript) => superscript = false,
3423            E::Start(C::Subscript, _) => subscript = true,
3424            E::End(C::Subscript) => subscript = false,
3425            E::Start(C::Insert, _) => underline = true,
3426            E::End(C::Insert) => underline = false,
3427            E::Start(C::Delete, _) => strikeout = true,
3428            E::End(C::Delete) => strikeout = false,
3429            // Highlight/mark and bare spans have no model field — keep the text.
3430            E::Start(C::Mark, _) | E::End(C::Mark) => {}
3431            E::Start(C::Span, _) | E::End(C::Span) => {}
3432            E::Start(C::Link(dst, _), _) => link_href = Some(dst.to_string()),
3433            E::End(C::Link(..)) => link_href = None,
3434            // Inline images. Djot writes display size as inline attributes
3435            // (`![alt](src){width=800 height=600}`), which jotdown hands over
3436            // on the `Start` event — verified against jotdown 0.10, including
3437            // quoted values and images mid-sentence.
3438            E::Start(C::Image(src, _), attrs) => {
3439                let attr_num = |key: &str| -> i64 {
3440                    attrs
3441                        .get_value(key)
3442                        .map(|v| v.to_string())
3443                        .and_then(|v| v.trim().parse::<i64>().ok())
3444                        .filter(|n| *n > 0)
3445                        .unwrap_or(0)
3446                };
3447                pending_image = Some(ParsedImage {
3448                    // The writer percent-encodes what the lexer would act on in a
3449                    // source; the source is a name, so it is decoded back.
3450                    src: crate::parser_tools::djot_escape::decode_djot_image_source(&src),
3451                    alt: String::new(),
3452                    width: attr_num("width"),
3453                    height: attr_num("height"),
3454                });
3455            }
3456            E::End(C::Image(..)) => {
3457                if let Some(img) = pending_image.take() {
3458                    push_image!(img);
3459                }
3460            }
3461
3462            // ── Footnote definitions ──
3463            //
3464            // The body is ordinary block content, so it is parsed by the
3465            // ordinary machinery: flush whatever inline run is open, note where
3466            // this definition's blocks start, and let them accumulate. `End`
3467            // lifts them back out. Nesting cannot occur — djot has no footnote
3468            // inside a footnote — so one mark suffices where the dropped
3469            // containers below need a depth counter.
3470            E::Start(C::Footnote { label }, _) => {
3471                if !current_spans.is_empty() {
3472                    djot_push_block(
3473                        &mut elements,
3474                        std::mem::take(&mut current_spans),
3475                        None,
3476                        cur_list_style.clone(),
3477                        cur_list_indent,
3478                        cur_list_prefix.clone(),
3479                        cur_list_suffix.clone(),
3480                        cur_list_start,
3481                        cur_marker.clone(),
3482                        false,
3483                        None,
3484                        blockquote_depth,
3485                        DjotBlockStyle::default(),
3486                    );
3487                }
3488                footnote_open = Some((label.to_string(), elements.len()));
3489            }
3490            E::End(C::Footnote { .. }) => {
3491                if !current_spans.is_empty() {
3492                    djot_push_block(
3493                        &mut elements,
3494                        std::mem::take(&mut current_spans),
3495                        None,
3496                        cur_list_style.clone(),
3497                        cur_list_indent,
3498                        cur_list_prefix.clone(),
3499                        cur_list_suffix.clone(),
3500                        cur_list_start,
3501                        cur_marker.clone(),
3502                        false,
3503                        None,
3504                        blockquote_depth,
3505                        DjotBlockStyle::default(),
3506                    );
3507                }
3508                if let Some((label, start)) = footnote_open.take() {
3509                    let blocks: Vec<ParsedBlock> = elements
3510                        .drain(start..)
3511                        .filter_map(|e| match e {
3512                            ParsedElement::Block(b) => Some(b),
3513                            // A table inside a footnote is not representable as
3514                            // note content; its cells would have to become
3515                            // blocks and lose their structure either way.
3516                            _ => None,
3517                        })
3518                        .collect();
3519                    elements.push(ParsedElement::FootnoteDefinition { label, blocks });
3520                }
3521            }
3522
3523            // ── Unrepresentable containers: drop the entire subtree ──
3524            E::Start(
3525                C::Math { .. }
3526                | C::RawBlock { .. }
3527                | C::RawInline { .. }
3528                | C::DescriptionList
3529                | C::DescriptionDetails
3530                | C::DescriptionTerm
3531                | C::Caption
3532                | C::LinkDefinition { .. },
3533                _,
3534            ) => skip_depth = 1,
3535
3536            // ── Text + atoms ──
3537            // A line ending inside a code span is kept by the parser as a line
3538            // break in the span's text, the one a paragraph's text can hold. It is
3539            // read as a hard break is, a new block with the list, quotation or
3540            // heading the line was in, so the model holds no line break inside a
3541            // block. Kept inside the span, the Djot writer saved the code run's
3542            // lines as blocks of their own, which reloaded as two blocks: a document
3543            // that did not settle into the Djot it saves as. Such a span is what an
3544            // older version wrote for a code run holding a line break, a pasted
3545            // `<pre><code>` passage above all, and reading its lines as blocks is how
3546            // that passage reads in the editor.
3547            E::Str(s) if code && !is_code_block && pending_image.is_none() && s.contains('\n') => {
3548                for (i, piece) in s.split('\n').enumerate() {
3549                    if i > 0 {
3550                        line_break!();
3551                    }
3552                    if !piece.is_empty() {
3553                        push_text!(piece);
3554                    }
3555                }
3556            }
3557            E::Str(s) => push_text!(s.as_ref()),
3558            E::Softbreak => push_text!(" "),
3559            E::LeftSingleQuote => push_text!("\u{2018}"),
3560            E::RightSingleQuote => push_text!("\u{2019}"),
3561            E::LeftDoubleQuote => push_text!("\u{201C}"),
3562            E::RightDoubleQuote => push_text!("\u{201D}"),
3563            E::Ellipsis => push_text!("\u{2026}"),
3564            E::EnDash => push_text!("\u{2013}"),
3565            E::EmDash => push_text!("\u{2014}"),
3566            E::NonBreakingSpace => push_text!("\u{00A0}"),
3567            E::Hardbreak => line_break!(),
3568            // A footnote reference. jotdown emits this purely syntactically —
3569            // it never checks that a matching `[^label]:` exists anywhere — so a
3570            // reference whose definition lives outside this document (the normal
3571            // state for a host that owns note bodies itself) arrives here just
3572            // the same, and must survive.
3573            E::FootnoteReference(label) => {
3574                let sp = ParsedSpan {
3575                    text: String::new(),
3576                    bold,
3577                    italic,
3578                    underline,
3579                    strikeout,
3580                    code,
3581                    superscript,
3582                    subscript,
3583                    link_href: link_href.clone(),
3584                    image: None,
3585                    footnote_ref: Some(label.to_string()),
3586                };
3587                if in_table_cell {
3588                    current_cell_spans.push(sp);
3589                } else {
3590                    current_spans.push(sp);
3591                }
3592            }
3593            // Symbols, escapes, blanklines, thematic breaks and dangling block
3594            // attributes carry no representable content.
3595            E::Symbol(_) => {}
3596            E::Escape | E::Blankline => {}
3597            E::ThematicBreak(_) | E::Attributes(_) => {}
3598
3599            // Ends of dropped containers (never reached at skip_depth 0) and any
3600            // future variants.
3601            _ => {}
3602        }
3603    }
3604
3605    // Flush any trailing inline content (defensive — Document End closes blocks).
3606    if !current_spans.is_empty() {
3607        djot_push_block(
3608            &mut elements,
3609            std::mem::take(&mut current_spans),
3610            current_heading.take(),
3611            cur_list_style.clone(),
3612            cur_list_indent,
3613            cur_list_prefix.clone(),
3614            cur_list_suffix.clone(),
3615            cur_list_start,
3616            cur_marker.clone(),
3617            is_code_block,
3618            code_language.take(),
3619            blockquote_depth,
3620            std::mem::take(&mut pending_style),
3621        );
3622    }
3623
3624    // An empty document still yields a single empty paragraph (matches
3625    // `parse_markdown`).
3626    if elements.is_empty() {
3627        djot_push_block(
3628            &mut elements,
3629            vec![ParsedSpan {
3630                text: String::new(),
3631                ..Default::default()
3632            }],
3633            None,
3634            None,
3635            0,
3636            String::new(),
3637            String::new(),
3638            None,
3639            None,
3640            false,
3641            None,
3642            0,
3643            DjotBlockStyle::default(),
3644        );
3645    }
3646
3647    elements
3648}
3649
3650#[cfg(test)]
3651mod tests {
3652    use super::*;
3653
3654    /// Helper: flatten parse_markdown output to blocks for tests that don't care about tables.
3655    fn parse_markdown_blocks(md: &str) -> Vec<ParsedBlock> {
3656        ParsedElement::flatten_to_blocks(parse_markdown(md))
3657    }
3658
3659    #[test]
3660    fn test_parse_markdown_simple_paragraph() {
3661        let blocks = parse_markdown_blocks("Hello **world**");
3662        assert_eq!(blocks.len(), 1);
3663        assert!(blocks[0].spans.len() >= 2);
3664        // "Hello " is plain, "world" is bold
3665        let plain_span = blocks[0]
3666            .spans
3667            .iter()
3668            .find(|s| s.text.contains("Hello"))
3669            .unwrap();
3670        assert!(!plain_span.bold);
3671        let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3672        assert!(bold_span.bold);
3673    }
3674
3675    #[test]
3676    fn test_parse_markdown_heading() {
3677        let blocks = parse_markdown_blocks("# Title");
3678        assert_eq!(blocks.len(), 1);
3679        assert_eq!(blocks[0].heading_level, Some(1));
3680        assert_eq!(blocks[0].spans[0].text, "Title");
3681    }
3682
3683    #[test]
3684    fn test_parse_markdown_list() {
3685        let blocks = parse_markdown_blocks("- item1\n- item2");
3686        assert!(blocks.len() >= 2);
3687        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3688        assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
3689    }
3690
3691    /// Helper: extract (is_table, blockquote_depth) per element for nesting assertions.
3692    fn element_depths(elements: &[ParsedElement]) -> Vec<(bool, u32)> {
3693        elements
3694            .iter()
3695            .map(|e| match e {
3696                ParsedElement::Block(b) => (false, b.blockquote_depth),
3697                ParsedElement::Table(t) => (true, t.blockquote_depth),
3698                // Definitions carry no blockquote nesting of their own; this
3699                // helper exists for the nesting assertions and never sees one.
3700                ParsedElement::FootnoteDefinition { .. } => (false, 0),
3701            })
3702            .collect()
3703    }
3704
3705    #[test]
3706    fn test_parse_markdown_table_in_blockquote_records_depth() {
3707        let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3708        assert_eq!(element_depths(&elements), vec![(true, 1)]);
3709    }
3710
3711    #[test]
3712    fn test_parse_markdown_text_then_table_in_blockquote() {
3713        let elements = parse_markdown("> Para\n>\n> | a | b |\n> |---|---|\n> | c | d |");
3714        assert_eq!(element_depths(&elements), vec![(false, 1), (true, 1)]);
3715    }
3716
3717    #[test]
3718    fn test_parse_markdown_table_after_blockquote_closes() {
3719        let elements = parse_markdown("> Para\n\n| a | b |\n|---|---|\n| c | d |");
3720        assert_eq!(element_depths(&elements), vec![(false, 1), (true, 0)]);
3721    }
3722
3723    #[test]
3724    fn test_parse_markdown_table_in_nested_blockquote() {
3725        let elements = parse_markdown(">> | a | b |\n>> |---|---|\n>> | c | d |");
3726        assert_eq!(element_depths(&elements), vec![(true, 2)]);
3727    }
3728
3729    #[test]
3730    fn test_parse_markdown_list_in_blockquote_records_depth() {
3731        let elements = parse_markdown("> - item1\n> - item2");
3732        let depths = element_depths(&elements);
3733        assert_eq!(depths, vec![(false, 1), (false, 1)]);
3734        for e in &elements {
3735            if let ParsedElement::Block(b) = e {
3736                assert_eq!(b.list_style, Some(ListStyle::Disc));
3737            }
3738        }
3739    }
3740
3741    #[test]
3742    fn test_parse_html_table_in_blockquote_records_depth() {
3743        let elements = parse_html_elements(
3744            "<blockquote><table><tr><th>A</th></tr><tr><td>x</td></tr></table></blockquote>",
3745        );
3746        assert_eq!(element_depths(&elements), vec![(true, 1)]);
3747    }
3748
3749    #[test]
3750    fn test_parse_html_table_after_blockquote() {
3751        let elements = parse_html_elements(
3752            "<blockquote><p>Para</p></blockquote><table><tr><td>X</td></tr></table>",
3753        );
3754        let depths = element_depths(&elements);
3755        // The blockquote paragraph carries depth 1; the table is outside (depth 0).
3756        assert!(depths.contains(&(false, 1)), "depths: {depths:?}");
3757        assert!(depths.contains(&(true, 0)), "depths: {depths:?}");
3758    }
3759
3760    #[test]
3761    fn test_flatten_to_blocks_propagates_blockquote_depth() {
3762        let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3763        let blocks = ParsedElement::flatten_to_blocks(elements);
3764        assert!(!blocks.is_empty());
3765        for b in &blocks {
3766            assert_eq!(b.blockquote_depth, 1);
3767        }
3768    }
3769
3770    #[test]
3771    fn test_parse_html_simple() {
3772        let blocks = parse_html("<p>Hello <b>world</b></p>");
3773        assert_eq!(blocks.len(), 1);
3774        assert!(blocks[0].spans.len() >= 2);
3775        let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3776        assert!(bold_span.bold);
3777    }
3778
3779    #[test]
3780    fn test_parse_html_multiple_paragraphs() {
3781        let blocks = parse_html("<p>A</p><p>B</p>");
3782        assert_eq!(blocks.len(), 2);
3783    }
3784
3785    #[test]
3786    fn test_parse_html_heading() {
3787        let blocks = parse_html("<h2>Subtitle</h2>");
3788        assert_eq!(blocks.len(), 1);
3789        assert_eq!(blocks[0].heading_level, Some(2));
3790    }
3791
3792    #[test]
3793    fn test_parse_html_list() {
3794        let blocks = parse_html("<ul><li>one</li><li>two</li></ul>");
3795        assert!(blocks.len() >= 2);
3796        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3797    }
3798
3799    #[test]
3800    fn test_parse_markdown_code_block() {
3801        let blocks = parse_markdown_blocks("```\nfn main() {}\n```");
3802        assert_eq!(blocks.len(), 1);
3803        assert!(blocks[0].is_code_block);
3804        assert!(blocks[0].spans[0].code);
3805        // pulldown-cmark appends a trailing \n to code block text — verify it's stripped
3806        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3807        assert_eq!(
3808            text, "fn main() {}",
3809            "code block text should not have trailing newline"
3810        );
3811    }
3812
3813    #[test]
3814    fn test_parse_markdown_nested_formatting() {
3815        let blocks = parse_markdown_blocks("***bold italic***");
3816        assert_eq!(blocks.len(), 1);
3817        let span = &blocks[0].spans[0];
3818        assert!(span.bold);
3819        assert!(span.italic);
3820    }
3821
3822    #[test]
3823    fn test_parse_markdown_link() {
3824        let blocks = parse_markdown_blocks("[click](http://example.com)");
3825        assert_eq!(blocks.len(), 1);
3826        let span = &blocks[0].spans[0];
3827        assert_eq!(span.text, "click");
3828        assert_eq!(span.link_href, Some("http://example.com".to_string()));
3829    }
3830
3831    #[test]
3832    fn test_parse_markdown_empty() {
3833        let blocks = parse_markdown_blocks("");
3834        assert_eq!(blocks.len(), 1);
3835        assert!(blocks[0].spans[0].text.is_empty());
3836    }
3837
3838    #[test]
3839    fn test_parse_html_empty() {
3840        let blocks = parse_html("");
3841        assert_eq!(blocks.len(), 1);
3842        assert!(blocks[0].spans[0].text.is_empty());
3843    }
3844
3845    #[test]
3846    fn test_parse_html_nested_formatting() {
3847        let blocks = parse_html("<p><b><i>bold italic</i></b></p>");
3848        assert_eq!(blocks.len(), 1);
3849        let span = &blocks[0].spans[0];
3850        assert!(span.bold);
3851        assert!(span.italic);
3852    }
3853
3854    #[test]
3855    fn test_parse_html_link() {
3856        let blocks = parse_html("<p><a href=\"http://example.com\">click</a></p>");
3857        assert_eq!(blocks.len(), 1);
3858        let span = &blocks[0].spans[0];
3859        assert_eq!(span.text, "click");
3860        assert_eq!(span.link_href, Some("http://example.com".to_string()));
3861    }
3862
3863    #[test]
3864    fn test_parse_html_ordered_list() {
3865        let blocks = parse_html("<ol><li>first</li><li>second</li></ol>");
3866        assert!(blocks.len() >= 2);
3867        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3868    }
3869
3870    #[test]
3871    fn test_parse_markdown_ordered_list() {
3872        let blocks = parse_markdown_blocks("1. first\n2. second");
3873        assert!(blocks.len() >= 2);
3874        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3875    }
3876
3877    #[test]
3878    fn test_parse_html_blockquote_nested() {
3879        let blocks = parse_html("<p>before</p><blockquote>quoted</blockquote><p>after</p>");
3880        assert!(blocks.len() >= 3);
3881    }
3882
3883    #[test]
3884    fn test_parse_block_styles_line_height() {
3885        let styles = parse_block_styles("line-height: 1.5");
3886        assert_eq!(styles.line_height, Some(1500));
3887    }
3888
3889    #[test]
3890    fn test_parse_block_styles_direction_rtl() {
3891        let styles = parse_block_styles("direction: rtl");
3892        assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3893    }
3894
3895    #[test]
3896    fn test_parse_block_styles_background_color() {
3897        let styles = parse_block_styles("background-color: #ff0000");
3898        assert_eq!(styles.background_color, Some("#ff0000".to_string()));
3899    }
3900
3901    #[test]
3902    fn test_parse_block_styles_white_space_pre() {
3903        let styles = parse_block_styles("white-space: pre");
3904        assert_eq!(styles.non_breakable_lines, Some(true));
3905    }
3906
3907    #[test]
3908    fn test_parse_block_styles_multiple() {
3909        let styles = parse_block_styles("line-height: 2.0; direction: rtl; background-color: blue");
3910        assert_eq!(styles.line_height, Some(2000));
3911        assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3912        assert_eq!(styles.background_color, Some("blue".to_string()));
3913    }
3914
3915    #[test]
3916    fn test_parse_html_block_styles_extracted() {
3917        let blocks = parse_html(
3918            r#"<p style="line-height: 1.5; direction: rtl; background-color: #ccc">text</p>"#,
3919        );
3920        assert_eq!(blocks.len(), 1);
3921        assert_eq!(blocks[0].line_height, Some(1500));
3922        assert_eq!(blocks[0].direction, Some(TextDirection::RightToLeft));
3923        assert_eq!(blocks[0].background_color, Some("#ccc".to_string()));
3924    }
3925
3926    /// The shape every word processor publishes on the clipboard: a newline
3927    /// after the opening tag, the prose hard-wrapped inside it, and a newline
3928    /// between every pair of blocks. None of it is text.
3929    #[test]
3930    fn test_parse_html_collapses_exporter_line_wrapping() {
3931        let blocks = parse_html(
3932            "<body>\n<p style=\"line-height: 200%\">\nfirst line wrapped\nhere</p>\n\
3933             <p>\nsecond para</p>\n</body>",
3934        );
3935        let texts: Vec<String> = blocks
3936            .iter()
3937            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
3938            .collect();
3939        assert_eq!(texts, vec!["first line wrapped here", "second para"]);
3940    }
3941
3942    /// The reported bug: pasting a whole LibreOffice book ended in one block
3943    /// holding every newline the exporter had written between its paragraphs —
3944    /// four thousand blank lines under the last sentence.
3945    #[test]
3946    fn test_parse_html_drops_whitespace_between_blocks() {
3947        let blocks = parse_html("<html>\n<body>\n<p>a</p>\n<p>b</p>\n</body>\n</html>");
3948        assert_eq!(blocks.len(), 2, "no block for the whitespace between them");
3949        assert!(
3950            blocks
3951                .iter()
3952                .all(|b| b.spans.iter().all(|s| !s.text.contains('\n'))),
3953            "no block keeps a literal newline"
3954        );
3955    }
3956
3957    /// A run split across inline elements is still one run, and it collapses to
3958    /// one space rather than disappearing between them.
3959    #[test]
3960    fn test_parse_html_collapses_across_span_boundaries() {
3961        let blocks = parse_html("<p>  <b>bold</b>\n  <i>italic</i>  </p>");
3962        assert_eq!(blocks.len(), 1);
3963        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3964        assert_eq!(text, "bold italic");
3965    }
3966
3967    /// U+00A0 is content, not layout — the space holding "Attention&nbsp;:"
3968    /// together in French must survive a paste.
3969    #[test]
3970    fn test_parse_html_keeps_no_break_space() {
3971        let blocks = parse_html("<p>\nAttention&nbsp;: ici</p>");
3972        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3973        assert_eq!(text, "Attention\u{a0}: ici");
3974    }
3975
3976    /// An empty paragraph is a paragraph. Only the whitespace *between* blocks
3977    /// goes; a block the writer opened stays, however little is in it.
3978    #[test]
3979    fn test_parse_html_keeps_empty_paragraph() {
3980        let blocks = parse_html("<body>\n<p>a</p>\n<p> </p>\n<p>b</p>\n</body>");
3981        assert_eq!(blocks.len(), 3);
3982        let text: String = blocks[1].spans.iter().map(|s| s.text.as_str()).collect();
3983        assert_eq!(text, "");
3984    }
3985
3986    /// `<pre>` and `white-space: pre` mean the source's spacing *is* the
3987    /// content. `nowrap` does not: it forbids wrapping and collapses runs like
3988    /// any other block, which is why it is not in the same test.
3989    #[test]
3990    fn test_parse_html_preserves_whitespace_in_pre() {
3991        // The newline straight after `<pre>` is the one the HTML parser itself
3992        // drops, per spec; every newline after that is content.
3993        let blocks = parse_html("<pre>\nfn main() {\n    let x = 1;\n}</pre>");
3994        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3995        assert_eq!(text, "fn main() {\n    let x = 1;\n}");
3996
3997        // Outside a code block a line break is a new block, and each line keeps its
3998        // spaces.
3999        let styled = parse_html("<p style=\"white-space: pre-wrap\">a\n  b</p>");
4000        let texts: Vec<String> = styled
4001            .iter()
4002            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4003            .collect();
4004        assert_eq!(texts, ["a", "  b"]);
4005
4006        let nowrap = parse_html("<p style=\"white-space: nowrap\">a\n  b</p>");
4007        let text: String = nowrap[0].spans.iter().map(|s| s.text.as_str()).collect();
4008        assert_eq!(text, "a b");
4009    }
4010
4011    /// Table cells are blocks too, and an exporter indents them just as
4012    /// generously as it indents paragraphs.
4013    #[test]
4014    fn test_parse_html_collapses_table_cell_whitespace() {
4015        let elements = parse_html_elements(
4016            "<table>\n<tr>\n<td>\n  one\n</td>\n<td>\n  two\n</td>\n</tr>\n</table>",
4017        );
4018        let table = elements
4019            .iter()
4020            .find_map(|e| match e {
4021                ParsedElement::Table(t) => Some(t),
4022                _ => None,
4023            })
4024            .expect("a table");
4025        let cells: Vec<String> = table.rows[0]
4026            .iter()
4027            .map(|c| c.spans.iter().map(|s| s.text.as_str()).collect())
4028            .collect();
4029        assert_eq!(cells, vec!["one", "two"]);
4030    }
4031
4032    #[test]
4033    fn test_parse_html_white_space_pre() {
4034        let blocks = parse_html(r#"<p style="white-space: pre">code</p>"#);
4035        assert_eq!(blocks.len(), 1);
4036        assert_eq!(blocks[0].non_breakable_lines, Some(true));
4037    }
4038
4039    #[test]
4040    fn test_parse_html_no_styles_returns_none() {
4041        let blocks = parse_html("<p>plain</p>");
4042        assert_eq!(blocks.len(), 1);
4043        assert_eq!(blocks[0].line_height, None);
4044        assert_eq!(blocks[0].direction, None);
4045        assert_eq!(blocks[0].background_color, None);
4046        assert_eq!(blocks[0].non_breakable_lines, None);
4047    }
4048
4049    #[test]
4050    fn test_parse_markdown_nested_list_indent() {
4051        let md = "- top\n  - nested\n    - deep";
4052        let blocks = parse_markdown_blocks(md);
4053        assert_eq!(blocks.len(), 3);
4054        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4055        assert_eq!(blocks[0].list_indent, 0);
4056        assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
4057        assert_eq!(blocks[1].list_indent, 1);
4058        assert_eq!(blocks[2].list_style, Some(ListStyle::Disc));
4059        assert_eq!(blocks[2].list_indent, 2);
4060    }
4061
4062    #[test]
4063    fn test_parse_markdown_nested_ordered_list_indent() {
4064        let md = "1. first\n   1. nested\n   2. nested2";
4065        let blocks = parse_markdown_blocks(md);
4066        assert_eq!(blocks.len(), 3);
4067        assert_eq!(blocks[0].list_indent, 0);
4068        assert_eq!(blocks[1].list_indent, 1);
4069        assert_eq!(blocks[2].list_indent, 1);
4070    }
4071
4072    #[test]
4073    fn test_parse_html_nested_list_indent() {
4074        let html = "<ul><li>top</li><ul><li>nested</li></ul></ul>";
4075        let blocks = parse_html(html);
4076        assert!(blocks.len() >= 2);
4077        assert_eq!(blocks[0].list_indent, 0);
4078        assert_eq!(blocks[1].list_indent, 1);
4079    }
4080
4081    #[test]
4082    fn test_parse_markdown_table() {
4083        let md = "| A | B |\n|---|---|\n| 1 | 2 |";
4084        let elements = parse_markdown(md);
4085        assert_eq!(elements.len(), 1);
4086        match &elements[0] {
4087            ParsedElement::Table(table) => {
4088                assert_eq!(table.header_rows, 1);
4089                assert_eq!(table.rows.len(), 2); // 1 header + 1 body
4090                // Header row
4091                assert_eq!(table.rows[0].len(), 2);
4092                assert_eq!(table.rows[0][0].spans[0].text, "A");
4093                assert_eq!(table.rows[0][1].spans[0].text, "B");
4094                // Body row
4095                assert_eq!(table.rows[1].len(), 2);
4096                assert_eq!(table.rows[1][0].spans[0].text, "1");
4097                assert_eq!(table.rows[1][1].spans[0].text, "2");
4098            }
4099            _ => panic!("Expected ParsedElement::Table"),
4100        }
4101    }
4102
4103    #[test]
4104    fn test_parse_markdown_table_with_formatting() {
4105        let md = "| **bold** | `code` | *italic* |\n|---|---|---|\n| ~~strike~~ | plain | [link](http://x.com) |";
4106        let elements = parse_markdown(md);
4107        assert_eq!(elements.len(), 1);
4108        match &elements[0] {
4109            ParsedElement::Table(table) => {
4110                assert_eq!(table.rows.len(), 2);
4111                // Header: bold cell
4112                assert!(table.rows[0][0].spans[0].bold);
4113                // Header: code cell
4114                assert!(table.rows[0][1].spans[0].code);
4115                // Header: italic cell
4116                assert!(table.rows[0][2].spans[0].italic);
4117                // Body: strikeout cell
4118                assert!(table.rows[1][0].spans[0].strikeout);
4119                // Body: link cell
4120                assert_eq!(
4121                    table.rows[1][2].spans[0].link_href,
4122                    Some("http://x.com".to_string())
4123                );
4124            }
4125            _ => panic!("Expected ParsedElement::Table"),
4126        }
4127    }
4128
4129    #[test]
4130    fn test_parse_markdown_mixed_content_with_table() {
4131        let md = "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter";
4132        let elements = parse_markdown(md);
4133        assert_eq!(elements.len(), 3);
4134        assert!(matches!(&elements[0], ParsedElement::Block(_)));
4135        assert!(matches!(&elements[1], ParsedElement::Table(_)));
4136        assert!(matches!(&elements[2], ParsedElement::Block(_)));
4137    }
4138}
4139
4140#[cfg(test)]
4141mod djot_tests {
4142    use super::*;
4143    use crate::entities::MarkerType;
4144
4145    fn blocks(d: &str) -> Vec<ParsedBlock> {
4146        ParsedElement::flatten_to_blocks(parse_djot(d, &DjotImportOptions::default()))
4147    }
4148
4149    fn first_span_with(b: &ParsedBlock, pred: impl Fn(&ParsedSpan) -> bool) -> &ParsedSpan {
4150        b.spans.iter().find(|s| pred(s)).expect("span not found")
4151    }
4152
4153    #[test]
4154    fn paragraph_bold_italic() {
4155        let b = blocks("normal *bold* _italic_");
4156        assert_eq!(b.len(), 1);
4157        assert!(first_span_with(&b[0], |s| s.text == "bold").bold);
4158        assert!(first_span_with(&b[0], |s| s.text == "italic").italic);
4159    }
4160
4161    #[test]
4162    fn heading_levels() {
4163        assert_eq!(blocks("# H1")[0].heading_level, Some(1));
4164        assert_eq!(blocks("### H3")[0].heading_level, Some(3));
4165        assert_eq!(blocks("###### H6")[0].heading_level, Some(6));
4166    }
4167
4168    #[test]
4169    fn unordered_bullet_styles_are_distinct() {
4170        assert_eq!(blocks("- a")[0].list_style, Some(ListStyle::Disc));
4171        assert_eq!(blocks("* a")[0].list_style, Some(ListStyle::Circle));
4172        assert_eq!(blocks("+ a")[0].list_style, Some(ListStyle::Square));
4173    }
4174
4175    #[test]
4176    fn ordered_delimiters() {
4177        let period = blocks("1. a");
4178        assert_eq!(period[0].list_style, Some(ListStyle::Decimal));
4179        assert_eq!(period[0].list_prefix, "");
4180        assert_eq!(period[0].list_suffix, ".");
4181
4182        let paren = blocks("1) a");
4183        assert_eq!(paren[0].list_suffix, ")");
4184        assert_eq!(paren[0].list_prefix, "");
4185
4186        let paren_paren = blocks("(1) a");
4187        assert_eq!(paren_paren[0].list_prefix, "(");
4188        assert_eq!(paren_paren[0].list_suffix, ")");
4189    }
4190
4191    #[test]
4192    fn task_list_markers() {
4193        let b = blocks("- [ ] a\n- [x] b");
4194        assert_eq!(b.len(), 2);
4195        assert_eq!(b[0].marker, Some(MarkerType::Unchecked));
4196        assert_eq!(b[1].marker, Some(MarkerType::Checked));
4197    }
4198
4199    #[test]
4200    fn code_block_with_language() {
4201        let b = blocks("```rust\nfn main() {}\n```");
4202        assert_eq!(b.len(), 1);
4203        assert!(b[0].is_code_block);
4204        assert_eq!(b[0].code_language.as_deref(), Some("rust"));
4205        let text: String = b[0].spans.iter().map(|s| s.text.as_str()).collect();
4206        assert_eq!(text, "fn main() {}");
4207    }
4208
4209    #[test]
4210    fn link_href() {
4211        let b = blocks("[text](http://example.com)");
4212        let s = first_span_with(&b[0], |s| s.text == "text");
4213        assert_eq!(s.link_href.as_deref(), Some("http://example.com"));
4214    }
4215
4216    #[test]
4217    fn superscript_subscript() {
4218        assert!(first_span_with(&blocks("a^b^")[0], |s| s.text == "b").superscript);
4219        assert!(first_span_with(&blocks("a~b~")[0], |s| s.text == "b").subscript);
4220    }
4221
4222    #[test]
4223    fn delete_insert_verbatim() {
4224        assert!(first_span_with(&blocks("{-x-}")[0], |s| s.text == "x").strikeout);
4225        assert!(first_span_with(&blocks("{+x+}")[0], |s| s.text == "x").underline);
4226        assert!(first_span_with(&blocks("`x`")[0], |s| s.text == "x").code);
4227    }
4228
4229    #[test]
4230    fn blockquote_depth() {
4231        let els = parse_djot("> quoted", &DjotImportOptions::default());
4232        match &els[0] {
4233            ParsedElement::Block(b) => assert_eq!(b.blockquote_depth, 1),
4234            _ => panic!("expected block"),
4235        }
4236    }
4237
4238    #[test]
4239    fn nested_list_indent() {
4240        // Djot nests a sub-list only when a blank line separates it from the
4241        // parent item and it is indented to the parent's content column
4242        // (2 spaces per level). Without the blank line the markers fold into
4243        // the paragraph as lazy continuation.
4244        let b = blocks("- a\n\n  - b\n\n    - c");
4245        assert_eq!(b.len(), 3);
4246        assert_eq!(b[0].list_indent, 0);
4247        assert_eq!(b[1].list_indent, 1);
4248        assert_eq!(b[2].list_indent, 2);
4249    }
4250
4251    #[test]
4252    fn table_parsed_as_table() {
4253        let els = parse_djot(
4254            "| a | b |\n|---|---|\n| c | d |",
4255            &DjotImportOptions::default(),
4256        );
4257        assert_eq!(els.len(), 1);
4258        match &els[0] {
4259            ParsedElement::Table(t) => {
4260                assert_eq!(t.header_rows, 1);
4261                assert_eq!(t.rows.len(), 2);
4262                assert_eq!(t.rows[0][0].spans[0].text, "a");
4263                assert_eq!(t.rows[1][1].spans[0].text, "d");
4264            }
4265            _ => panic!("expected table"),
4266        }
4267    }
4268
4269    #[test]
4270    fn smart_punctuation_normalised_to_unicode() {
4271        let text: String = blocks("a... b---c")[0]
4272            .spans
4273            .iter()
4274            .map(|s| s.text.as_str())
4275            .collect();
4276        assert!(text.contains('\u{2026}'), "ellipsis: {text:?}");
4277        assert!(text.contains('\u{2014}'), "em dash: {text:?}");
4278    }
4279
4280    #[test]
4281    fn unrepresentable_constructs_dropped_without_leaking_text() {
4282        // Thematic break between two paragraphs: no extra block, no stray text.
4283        let b = blocks("para1\n\n---\n\npara2");
4284        assert_eq!(b.len(), 2);
4285        assert_eq!(
4286            b[0].spans
4287                .iter()
4288                .map(|s| s.text.as_str())
4289                .collect::<String>(),
4290            "para1"
4291        );
4292        assert_eq!(
4293            b[1].spans
4294                .iter()
4295                .map(|s| s.text.as_str())
4296                .collect::<String>(),
4297            "para2"
4298        );
4299
4300        // Fenced div is unwrapped: its content survives, the fence does not.
4301        let d = blocks(":::\ninside\n:::");
4302        let joined: String = d
4303            .iter()
4304            .flat_map(|b| b.spans.iter())
4305            .map(|s| s.text.as_str())
4306            .collect();
4307        assert_eq!(joined, "inside");
4308
4309        // Inline math content is dropped, surrounding text kept.
4310        let m = blocks("before $`E=mc^2` after");
4311        let joined: String = m
4312            .iter()
4313            .flat_map(|b| b.spans.iter())
4314            .map(|s| s.text.as_str())
4315            .collect();
4316        assert!(joined.contains("before"), "{joined:?}");
4317        assert!(joined.contains("after"), "{joined:?}");
4318        assert!(!joined.contains("E=mc"), "math leaked: {joined:?}");
4319    }
4320
4321    #[test]
4322    fn empty_document_yields_one_empty_block() {
4323        let b = blocks("");
4324        assert_eq!(b.len(), 1);
4325        assert!(b[0].spans.iter().all(|s| s.text.is_empty()));
4326    }
4327
4328    #[test]
4329    fn block_attributes_parse_into_block() {
4330        let b = blocks(
4331            "{alignment=center line_height=1500 direction=rtl non_breakable_lines=true background_color=\"#ff0000\"}\nhello",
4332        );
4333        assert_eq!(b.len(), 1);
4334        assert_eq!(b[0].alignment, Some(Alignment::Center));
4335        assert_eq!(b[0].line_height, Some(1500));
4336        assert_eq!(b[0].direction, Some(TextDirection::RightToLeft));
4337        assert_eq!(b[0].non_breakable_lines, Some(true));
4338        assert_eq!(b[0].background_color, Some("#ff0000".to_string()));
4339    }
4340
4341    #[test]
4342    fn spacing_block_attributes_parse_into_block() {
4343        // `top_margin` / `text_indent` let one block override the document-wide
4344        // paragraph spacing and first-line indent — what a scene break needs for
4345        // the paragraph that follows it.
4346        let b = blocks("{top_margin=24 text_indent=0}\nhello");
4347        assert_eq!(b.len(), 1);
4348        assert_eq!(b[0].top_margin, Some(24));
4349        assert_eq!(b[0].text_indent, Some(0));
4350    }
4351
4352    #[test]
4353    fn a_zero_text_indent_is_distinct_from_an_absent_one() {
4354        // The whole point of the attribute: `Some(0)` means "explicitly no
4355        // indent", which must not collapse to `None` ("use the document
4356        // default") — otherwise a scene break could not suppress the indent.
4357        let explicit = blocks("{text_indent=0}\nhello");
4358        let absent = blocks("hello");
4359        assert_eq!(explicit[0].text_indent, Some(0));
4360        assert_eq!(absent[0].text_indent, None);
4361        assert!(!explicit[0].is_inline_only());
4362        assert!(absent[0].is_inline_only());
4363    }
4364
4365    #[test]
4366    fn spacing_block_attributes_respect_import_options() {
4367        let src = "{top_margin=24 text_indent=0}\nhello";
4368        let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4369        assert_eq!(b[0].top_margin, None);
4370        assert_eq!(b[0].text_indent, None);
4371    }
4372
4373    #[test]
4374    fn block_attributes_on_heading() {
4375        let b = blocks("{alignment=right}\n# Title");
4376        assert_eq!(b[0].heading_level, Some(1));
4377        assert_eq!(b[0].alignment, Some(Alignment::Right));
4378    }
4379
4380    #[test]
4381    fn block_attributes_respect_import_options() {
4382        // With every optional attribute disabled, the `{…}` block attributes are
4383        // parsed and discarded — only the core paragraph survives.
4384        let src = "{alignment=center line_height=1500}\nhello";
4385        let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4386        assert_eq!(b[0].alignment, None);
4387        assert_eq!(b[0].line_height, None);
4388        assert_eq!(
4389            b[0].spans
4390                .iter()
4391                .map(|s| s.text.as_str())
4392                .collect::<String>(),
4393            "hello"
4394        );
4395    }
4396
4397    #[test]
4398    fn list_item_block_attributes_are_dropped() {
4399        // Block attributes only bind to standalone paragraphs/headings; a list
4400        // item normalises them away (symmetric with the exporter).
4401        let b = blocks("{alignment=center}\n- item");
4402        assert!(b.iter().all(|blk| blk.alignment.is_none()));
4403    }
4404
4405    #[test]
4406    fn unknown_alignment_value_is_ignored() {
4407        let b = blocks("{alignment=sideways}\nhello");
4408        assert_eq!(b[0].alignment, None);
4409    }
4410}
4411
4412// ─── Cheap plain-text extraction ─────────────────────────────────────
4413
4414/// The `U+FFFC OBJECT REPLACEMENT CHARACTER` that stands for a table in the document's
4415/// text.
4416///
4417/// A table is not prose, but it *occupies a position* in the flow: the import mirrors this
4418/// single sentinel into the rope where the table sits
4419/// (`rope_helpers::rope_append_table_anchor`), then the cells as ordinary blocks. Anything
4420/// reconstructing the text a search runs against has to reproduce it, or every offset after
4421/// the first table is short by the two characters (the sentinel and its separator) that the
4422/// document really holds there.
4423pub const TABLE_ANCHOR: &str = "\u{FFFC}";
4424
4425/// One span's contribution to the addressable text.
4426///
4427/// An inline object carries no prose but **does** occupy one `U+FFFC` in the
4428/// document (`format_runs_from_spans` mirrors it there), so it has to occupy one
4429/// here too. Leaving it out makes this string shorter than the document it
4430/// claims to be byte-identical to, and every offset past the object — every
4431/// search hit, every comment anchor — lands a character early.
4432fn span_prose(span: &ParsedSpan, out: &mut String) {
4433    if span.image.is_some() || span.footnote_ref.is_some() {
4434        out.push('\u{FFFC}');
4435        return;
4436    }
4437    out.push_str(&span.text);
4438}
4439
4440fn block_prose(block: &ParsedBlock) -> String {
4441    let mut prose = String::new();
4442    for span in &block.spans {
4443        span_prose(span, &mut prose);
4444    }
4445    prose
4446}
4447
4448fn cell_prose(cell: &ParsedTableCell) -> String {
4449    let mut prose = String::new();
4450    for span in &cell.spans {
4451        span_prose(span, &mut prose);
4452    }
4453    prose
4454}
4455
4456/// The prose of a Djot document, with no entities, no store, and no threads.
4457///
4458/// [`parse_djot`] and [`ParsedElement::flatten_to_blocks`] were both already `pub`;
4459/// nothing chained them. This does, and that is the whole trick: it stops at the
4460/// *parse*, where a full import goes on to create a `Block` entity per paragraph, list
4461/// item and table cell, mirror each into the rope, and write its format runs.
4462///
4463/// # Why a project-wide search needs this
4464///
4465/// A host app searching a manuscript must ask "does this scene contain that word" of
4466/// **thousands** of Djot rows, on every keystroke. Doing that by importing each one into
4467/// a document is not a slow feature, it is a frozen app.
4468///
4469/// And searching the Djot *source* instead — the tempting shortcut — is simply wrong:
4470/// the source is markup. `http` matches inside a link's URL, `*` matches an emphasis
4471/// marker, and an occurrence count taken from the source does not agree with what a
4472/// replace re-derives inside the parsed document. Where a replace guards itself with
4473/// "the text moved under me, skip this field", a count taken from markup makes that
4474/// guard fire on perfectly good rows.
4475///
4476/// # The contract
4477///
4478/// The result is **byte-identical to the text the document searches** for the same Djot:
4479/// each block's spans concatenated, blocks joined by a single `\n`, and a table announced
4480/// by its [`TABLE_ANCHOR`] sentinel — exactly the string the import mirrors into the rope
4481/// and that `build_full_text_via_store` recomposes. So an offset found here is an offset
4482/// the document agrees with. On a live document the same string is served by
4483/// `TextDocument::to_addressable_text()`, which reads it off the search plumbing itself.
4484///
4485/// A property in `djot_roundtrip_tests` pins that across the whole generated feature set;
4486/// without it this would be a second, silently-diverging definition of "the text".
4487///
4488/// **Known exception: a footnote *definition*'s body.** This function treats it as out of
4489/// flow and omits it — matching `character_count()`, which does not count it — but the
4490/// live document mirrors its blocks into the rope, so today an in-document search DOES
4491/// run over the note's body and `to_addressable_text()` includes it. The live document
4492/// holds every body after the main text, so this string is the start of that one, and
4493/// every offset of the main text agrees between the two; they differ by the bodies at
4494/// the end. Which side is wrong is an open product question (should search see notes?),
4495/// pinned as current behaviour in
4496/// `addressable_text_tests::footnote_bodies_are_searched_in_the_live_document`.
4497///
4498/// ⚠ It is **not** the same as `TextDocument::to_plain_text()`. That is the human-readable
4499/// *export* — same prose, same order, but with every object anchor omitted, so its offsets
4500/// drift by two characters per preceding table. The authority is what a search sees,
4501/// because that is what a replace edits.
4502/// (`to_plain_text` once also *ordered* a blockquote's prose differently; that was a bug,
4503/// fixed and pinned by `plain_text_order_tests` — see
4504/// `claude_reviews/text-document-plain-text-ordering.md`. The anchors are now the two
4505/// views' only difference.)
4506pub fn djot_to_plain_text(djot: &str, options: &DjotImportOptions) -> String {
4507    // Deliberately NOT `ParsedElement::flatten_to_blocks`: that helper drops a table's
4508    // anchor and yields only its cells, which would silently shift every offset in a
4509    // document containing a table by the two characters the document actually holds there.
4510    let elements = parse_djot(djot, options);
4511
4512    // Sized from the source: the prose is always shorter than the markup that carries it,
4513    // so this allocates once and never grows.
4514    let mut out = String::with_capacity(djot.len());
4515
4516    // The separator is decided by "is this the first block", NOT by "is the output still
4517    // empty". An EMPTY block (an empty code fence, say) is still a block: the document
4518    // holds an empty line for it, and an emptiness test would swallow both the block and
4519    // its separator, shifting every offset after it by one.
4520    let mut first = true;
4521    let push = |text: &str, out: &mut String, first: &mut bool| {
4522        if *first {
4523            *first = false;
4524        } else {
4525            out.push('\n');
4526        }
4527        out.push_str(text);
4528    };
4529
4530    for element in &elements {
4531        match element {
4532            ParsedElement::Block(block) => {
4533                push(&block_prose(block), &mut out, &mut first);
4534            }
4535            // A note's body is **out of flow**: it is not laid out where its
4536            // definition was written, it is not part of a copied fragment, and
4537            // the document does not count its characters. So it is not part of
4538            // the addressable text either — which is the one thing that has to
4539            // stay true here, since `character_count()` and this string are
4540            // compared directly.
4541            //
4542            // Its blocks do live in the rope, because they have to live
4543            // somewhere; they are simply not addressable prose.
4544            ParsedElement::FootnoteDefinition { .. } => {}
4545            ParsedElement::Table(table) => {
4546                // The import mirrors a table into the rope as a lone `U+FFFC` sentinel
4547                // followed by its cells, one per block (`rope_append_table_anchor`). The
4548                // sentinel occupies a real position in the text the document searches, so
4549                // it has to occupy one here too — otherwise every offset after a table is
4550                // short by two characters, and a snippet taken from this string would be
4551                // sliced in the wrong place.
4552                push(TABLE_ANCHOR, &mut out, &mut first);
4553                // A row shorter than the widest holds empty cells in the document, one
4554                // position each: the import completes it (see `ParsedTable::column_count`).
4555                let columns = table.column_count();
4556                for row in &table.rows {
4557                    for column in 0..columns {
4558                        let prose = row.get(column).map(cell_prose).unwrap_or_default();
4559                        push(&prose, &mut out, &mut first);
4560                    }
4561                }
4562            }
4563        }
4564    }
4565    out
4566}
4567
4568#[cfg(test)]
4569mod html_footnote_tests {
4570    use super::*;
4571
4572    /// The reason this exists at all: a producer cannot smuggle a reference
4573    /// through as text, because the Djot escaper neutralises `[`, `^` and `]`.
4574    /// Pinned here so the escaping rule and this workaround cannot drift apart.
4575    #[test]
4576    fn a_literal_reference_in_text_would_be_escaped_into_prose() {
4577        let blocks = parse_html("<p>The ferry.[^1]</p>");
4578        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4579        assert!(
4580            text.contains("[^1]"),
4581            "the parser keeps it as characters: {text:?}"
4582        );
4583        assert!(
4584            blocks[0].spans.iter().all(|s| s.footnote_ref.is_none()),
4585            "text is text — it must not become a reference by accident"
4586        );
4587    }
4588
4589    #[test]
4590    fn an_attributed_element_becomes_a_real_footnote_reference() {
4591        let blocks = parse_html(
4592            r#"<p>The ferry was late.<sup data-footnote-ref="3"></sup> She waited.</p>"#,
4593        );
4594        let refs: Vec<&str> = blocks
4595            .iter()
4596            .flat_map(|b| b.spans.iter())
4597            .filter_map(|s| s.footnote_ref.as_deref())
4598            .collect();
4599        assert_eq!(refs, vec!["3"]);
4600
4601        // And it carries no text of its own, so it cannot be counted as words or
4602        // matched by a search.
4603        let marker = blocks[0]
4604            .spans
4605            .iter()
4606            .find(|s| s.footnote_ref.is_some())
4607            .expect("the reference span");
4608        assert!(marker.text.is_empty(), "{marker:?}");
4609    }
4610
4611    /// The attribute is the contract, not the tag: a producer may use whatever
4612    /// element renders sensibly in a browser.
4613    #[test]
4614    fn any_element_carrying_the_attribute_works() {
4615        for html in [
4616            r#"<p>a<sup data-footnote-ref="1"></sup></p>"#,
4617            r#"<p>a<span data-footnote-ref="1"></span></p>"#,
4618            r##"<p>a<a data-footnote-ref="1" href="#fn1"></a></p>"##,
4619        ] {
4620            let blocks = parse_html(html);
4621            assert_eq!(
4622                blocks
4623                    .iter()
4624                    .flat_map(|b| b.spans.iter())
4625                    .filter_map(|s| s.footnote_ref.as_deref())
4626                    .collect::<Vec<_>>(),
4627                vec!["1"],
4628                "failed for {html}"
4629            );
4630        }
4631    }
4632
4633    #[test]
4634    fn an_empty_or_absent_label_is_not_a_reference() {
4635        for html in [
4636            r#"<p>a<sup data-footnote-ref=""></sup></p>"#,
4637            r#"<p>a<sup></sup></p>"#,
4638        ] {
4639            let blocks = parse_html(html);
4640            assert!(
4641                blocks
4642                    .iter()
4643                    .flat_map(|b| b.spans.iter())
4644                    .all(|s| s.footnote_ref.is_none()),
4645                "failed for {html}"
4646            );
4647        }
4648    }
4649}
4650
4651#[cfg(test)]
4652mod html_depth_tests {
4653    //! HTML nested more deeply than the reader follows, and HTML table cells.
4654
4655    use super::*;
4656
4657    /// The text of each block, in order.
4658    fn block_texts(blocks: &[ParsedBlock]) -> Vec<String> {
4659        blocks
4660            .iter()
4661            .map(|block| block.spans.iter().map(|span| span.text.as_str()).collect())
4662            .collect()
4663    }
4664
4665    /// An HTML list nested past the deepest level an insertion keeps comes in with every
4666    /// item, in its order, the deeper ones side by side at that level. The reader stopped
4667    /// descending at its recursion limit and dropped every item below it (the eighty-sixth
4668    /// level on).
4669    #[test]
4670    fn a_deep_html_list_keeps_every_item_at_the_deepest_level() {
4671        use crate::parser_tools::list_depth::MAX_LIST_INDENT;
4672        let depth = 300;
4673        let mut html: String = (0..depth).map(|i| format!("<ol><li>item {i}")).collect();
4674        html.push_str(&"</li></ol>".repeat(depth));
4675        let blocks = parse_html(&html);
4676        let items: Vec<(String, u32)> = blocks
4677            .iter()
4678            .filter(|block| block.list_style.is_some())
4679            .map(|block| {
4680                let text: String = block.spans.iter().map(|span| span.text.as_str()).collect();
4681                (text, block.list_indent)
4682            })
4683            .collect();
4684        let expected: Vec<(String, u32)> = (0..depth)
4685            .map(|i| (format!("item {i}"), (i as i64).min(MAX_LIST_INDENT) as u32))
4686            .collect();
4687        assert_eq!(items, expected);
4688    }
4689
4690    /// Text outside any item of a list nested too deeply stays, after the items.
4691    #[test]
4692    fn text_in_a_deep_html_list_outside_its_items_is_kept() {
4693        let depth = 40;
4694        let mut html: String = (0..depth).map(|i| format!("<ul><li>item {i}")).collect();
4695        html.push_str("<ul>loose words<li>last</li></ul>");
4696        html.push_str(&"</li></ul>".repeat(depth));
4697        let texts = block_texts(&parse_html(&html)).join(" ");
4698        assert!(texts.contains("loose words"), "{texts:?}");
4699        assert!(texts.contains("last"), "{texts:?}");
4700        assert!(texts.contains("item 39"), "{texts:?}");
4701    }
4702
4703    /// Past the depth the reader follows, it used to keep nothing: the words at the bottom
4704    /// of three hundred nested `<div>`s, or quotations, were dropped from the paste.
4705    #[test]
4706    fn html_nested_past_the_reader_depth_keeps_its_words() {
4707        for (open, close) in [("<div>", "</div>"), ("<blockquote>", "</blockquote>")] {
4708            let html = format!(
4709                "<p>Top.</p>{}<p>Deep <b>words</b></p><p>here.</p>{}<p>End.</p>",
4710                open.repeat(300),
4711                close.repeat(300)
4712            );
4713            let texts = block_texts(&parse_html(&html)).join(" ");
4714            let words: Vec<&str> = texts.split_whitespace().collect();
4715            assert_eq!(
4716                words,
4717                ["Top.", "Deep", "words", "here.", "End."],
4718                "for {open}"
4719            );
4720        }
4721    }
4722
4723    /// The same inside a table cell, read as one paragraph.
4724    #[test]
4725    fn a_table_cell_nested_past_the_reader_depth_keeps_its_words() {
4726        let html = format!(
4727            "<table><tr><td>{}deep words{}</td></tr></table>",
4728            "<span>".repeat(300),
4729            "</span>".repeat(300)
4730        );
4731        let elements = parse_html_elements(&html);
4732        let Some(ParsedElement::Table(table)) = elements.first() else {
4733            panic!("a table: {elements:?}");
4734        };
4735        let text: String = table.rows[0][0]
4736            .spans
4737            .iter()
4738            .map(|span| span.text.as_str())
4739            .collect();
4740        assert_eq!(text, "deep words");
4741    }
4742
4743    /// A cell is one paragraph, so its line breaks and paragraphs are spaces between
4744    /// words. The HTML writer puts a cell's two paragraphs out as `One<br/>Two`, and the
4745    /// reader read them back as the one word `OneTwo`.
4746    #[test]
4747    fn line_breaks_and_paragraphs_in_an_html_cell_separate_words() {
4748        for cell in [
4749            "One<br/>Two",
4750            "<p>One</p><p>Two</p>",
4751            "<ul><li>One</li><li>Two</li></ul>",
4752        ] {
4753            let elements = parse_html_elements(&format!("<table><tr><td>{cell}</td></tr></table>"));
4754            let Some(ParsedElement::Table(table)) = elements.first() else {
4755                panic!("a table: {elements:?}");
4756            };
4757            let text: String = table.rows[0][0]
4758                .spans
4759                .iter()
4760                .map(|span| span.text.as_str())
4761                .collect();
4762            assert_eq!(text, "One Two", "for {cell}");
4763        }
4764    }
4765}