Skip to main content

common/parser_tools/
content_parser.rs

1use crate::entities::{Alignment, ListStyle, MarkerType, SemanticRole, TextDirection};
2use crate::parser_tools::djot_options::DjotImportOptions;
3
4/// An inline image recovered by a parser, before it becomes an `ImageAnchor`.
5///
6/// `src` is whatever the source document pointed at — a relative path, a bare
7/// name, a URL. Parsers do not resolve it; that is the embedding application's
8/// job, and this crate never touches the filesystem.
9#[derive(Debug, Clone, Default, PartialEq, Eq)]
10pub struct ParsedImage {
11    pub src: String,
12    pub alt: String,
13    /// Display size in pixels, `0` when the source did not state one.
14    ///
15    /// Djot and HTML both carry these as attributes (`{width=800}`,
16    /// `<img width=…>`); Markdown has no syntax for them, so a Markdown import
17    /// leaves both zero and the caller supplies intrinsic dimensions.
18    pub width: i64,
19    pub height: i64,
20}
21
22/// A parsed inline span with formatting info.
23///
24/// A span carries *either* text or an image, never both: an image span's `text`
25/// is empty and its description lives in [`ParsedImage::alt`], so alt text
26/// cannot leak into the block's prose (and therefore cannot be counted as
27/// manuscript words or matched by a search).
28#[derive(Debug, Clone, Default)]
29pub struct ParsedSpan {
30    pub text: String,
31    pub bold: bool,
32    pub italic: bool,
33    pub underline: bool,
34    pub strikeout: bool,
35    pub code: bool,
36    /// Superscript (djot `^x^`). Maps to `CharVerticalAlignment::SuperScript`.
37    pub superscript: bool,
38    /// Subscript (djot `~x~`). Maps to `CharVerticalAlignment::SubScript`.
39    pub subscript: bool,
40    pub link_href: Option<String>,
41    /// Set when this span is an inline image rather than text.
42    pub image: Option<ParsedImage>,
43    /// Set when this span is a footnote *reference* rather than text — the
44    /// label naming the note, never the number a reader sees.
45    pub footnote_ref: Option<String>,
46}
47
48/// A parsed table cell containing inline spans.
49#[derive(Debug, Clone)]
50pub struct ParsedTableCell {
51    pub spans: Vec<ParsedSpan>,
52}
53
54/// A parsed table extracted from markdown or HTML.
55#[derive(Debug, Clone)]
56pub struct ParsedTable {
57    /// Number of header rows (typically 1 for markdown tables).
58    pub header_rows: usize,
59    /// All rows (header + body), each containing cells with their inline spans.
60    pub rows: Vec<Vec<ParsedTableCell>>,
61    /// Blockquote nesting depth at the point the table appeared
62    /// (0 = not inside a blockquote), mirroring `ParsedBlock::blockquote_depth`.
63    pub blockquote_depth: u32,
64    /// How many of those quotations open at the table, mirroring
65    /// `ParsedBlock::blockquote_opens`.
66    pub blockquote_opens: u32,
67}
68
69impl ParsedTable {
70    /// How many columns the table needs to hold every cell it has: as many as its widest
71    /// row has cells. A row shorter than that is completed with empty cells by whoever
72    /// builds the table.
73    ///
74    /// The first row does not say it. The rows of a Djot table differ in length when their
75    /// source does, and an HTML table's first row is often one heading cell spanning every
76    /// column: a table sized from its first row kept the cells of a longer row in the model,
77    /// where they showed, while every export writes a row up to the table's width only, so
78    /// the first save dropped them.
79    pub fn column_count(&self) -> usize {
80        self.rows.iter().map(Vec::len).max().unwrap_or(0)
81    }
82}
83
84/// How many cells a table may reach once its short rows are completed (see
85/// [`ParsedTable::column_count`]), however few it has of its own: a table this size loads
86/// in a moment.
87const SQUARED_CELLS_ALWAYS: usize = 4096;
88
89/// How many times the cells it has of its own a larger table may reach once its short
90/// rows are completed.
91///
92/// A table whose rows differ in length because its source wrote them so, a wide heading
93/// row over rows of one cell or a few, grows to about as many times its own cells as it has
94/// columns: a dozen or so for a table a writer makes. The tables that made hundreds of
95/// thousands of cells out of a few kilobytes grow hundreds of times. At four times, a table
96/// of ten columns heading five hundred rows of one cell was read as paragraphs, its grid
97/// lost for a table that loads in a few milliseconds.
98const SQUARED_CELLS_PER_OWN_CELL: usize = 16;
99
100/// Push `table`, which has `own_cells` cells of its own in its source, onto `elements`: as
101/// a table, or as the paragraphs of its cells when completing its short rows would make it
102/// many times that.
103///
104/// Every row of a loaded table is completed to the widest one. A few kilobytes of a table
105/// whose first row is wide and whose other rows hold one cell each, or of HTML cells each
106/// spanning a thousand columns, made hundreds of thousands of empty cells, and held a load
107/// or a paste for seconds. Such a table is read as the paragraphs of its cells that hold
108/// anything, in reading order, at the table's quotation depth: every word stays, the grid
109/// goes.
110fn push_table(elements: &mut Vec<ParsedElement>, table: ParsedTable, own_cells: usize) {
111    let squared = table.rows.len().saturating_mul(table.column_count());
112    let allowed = SQUARED_CELLS_ALWAYS.max(own_cells.saturating_mul(SQUARED_CELLS_PER_OWN_CELL));
113    if squared <= allowed {
114        elements.push(ParsedElement::Table(table));
115        return;
116    }
117    elements.extend(
118        ParsedElement::flatten_to_blocks(vec![ParsedElement::Table(table)])
119            .into_iter()
120            .filter(|block| spans_carry_content(&block.spans))
121            .map(ParsedElement::Block),
122    );
123}
124
125/// Record that a quotation opened at the first block or table of `elements` from `first`
126/// on: the quotation that began where `elements` stood at `first` and closes now. Nothing
127/// when the quotation holds none.
128fn mark_quotation_opened(elements: &mut [ParsedElement], first: usize) {
129    let Some(rest) = elements.get_mut(first..) else {
130        return;
131    };
132    for element in rest {
133        match element {
134            ParsedElement::Block(block) => {
135                block.blockquote_opens += 1;
136                return;
137            }
138            ParsedElement::Table(table) => {
139                table.blockquote_opens += 1;
140                return;
141            }
142            ParsedElement::FootnoteDefinition { .. } => {}
143        }
144    }
145}
146
147/// The cells a table's rows hold, for [`push_table`].
148fn cells_in_rows(rows: &[Vec<ParsedTableCell>]) -> usize {
149    rows.iter().map(Vec::len).sum()
150}
151
152/// A parsed element: either a block or a table.
153#[derive(Debug, Clone)]
154pub enum ParsedElement {
155    Block(ParsedBlock),
156    Table(ParsedTable),
157    /// A footnote definition: the label its references name, and the blocks
158    /// making up its body.
159    ///
160    /// Separate from `Block` because a definition is not part of the flow it was
161    /// written in — it belongs wherever the output format puts notes, which the
162    /// importer expresses by giving it a detached frame of its own.
163    FootnoteDefinition {
164        label: String,
165        blocks: Vec<ParsedBlock>,
166    },
167}
168
169impl ParsedElement {
170    /// Extract blocks, flattening tables into one block per cell.
171    /// Use when table structure is not needed.
172    pub fn flatten_to_blocks(elements: Vec<ParsedElement>) -> Vec<ParsedBlock> {
173        let mut blocks = Vec::new();
174        for elem in elements {
175            match elem {
176                ParsedElement::Block(b) => blocks.push(b),
177                // A definition is not part of the flow it was written in — it
178                // belongs wherever the format puts notes. Flattening it in
179                // would splice a note's body into the prose at the point the
180                // definition happened to be typed.
181                ParsedElement::FootnoteDefinition { .. } => {}
182                ParsedElement::Table(t) => {
183                    for row in t.rows {
184                        for cell in row {
185                            blocks.push(ParsedBlock {
186                                spans: cell.spans,
187                                heading_level: None,
188                                list_style: None,
189                                list_indent: 0,
190                                list_prefix: String::new(),
191                                list_suffix: String::new(),
192                                list_start: None,
193                                marker: None,
194                                is_code_block: false,
195                                code_language: None,
196                                blockquote_depth: t.blockquote_depth,
197                                blockquote_opens: 0,
198                                line_height: None,
199                                non_breakable_lines: None,
200                                page_break_before: None,
201                                front_matter: None,
202                                direction: None,
203                                background_color: None,
204                                alignment: None,
205                                top_margin: None,
206                                text_indent: None,
207                                semantic_role: None,
208                            });
209                        }
210                    }
211                }
212            }
213        }
214        if blocks.is_empty() {
215            blocks.push(ParsedBlock {
216                spans: vec![ParsedSpan {
217                    text: String::new(),
218                    ..Default::default()
219                }],
220                heading_level: None,
221                list_style: None,
222                list_indent: 0,
223                list_prefix: String::new(),
224                list_suffix: String::new(),
225                list_start: None,
226                marker: None,
227                is_code_block: false,
228                code_language: None,
229                blockquote_depth: 0,
230                blockquote_opens: 0,
231                line_height: None,
232                non_breakable_lines: None,
233                page_break_before: None,
234                front_matter: None,
235                direction: None,
236                background_color: None,
237                alignment: None,
238                top_margin: None,
239                text_indent: None,
240                semantic_role: None,
241            });
242        }
243        blocks
244    }
245}
246
247/// A parsed block (paragraph, heading, list item, code block)
248///
249/// `Default` is an unformatted, empty paragraph — the shape a caller wants when
250/// it has prose and no structure to give it. Deriving it also means a field
251/// added later reaches those callers as "absent" rather than as a compile error
252/// they would fix by guessing a value.
253#[derive(Debug, Clone, Default)]
254pub struct ParsedBlock {
255    pub spans: Vec<ParsedSpan>,
256    pub heading_level: Option<i64>,
257    pub list_style: Option<ListStyle>,
258    pub list_indent: u32,
259    /// Ordered-list delimiter prefix (e.g. `"("` for djot `(1)` lists; empty
260    /// otherwise). Stored on the `List` entity for round-trip fidelity.
261    pub list_prefix: String,
262    /// Ordered-list delimiter suffix (`"."` for `1.`, `")"` for `1)`/`(1)`;
263    /// empty for unordered lists).
264    pub list_suffix: String,
265    /// The number the first item of the list this item belongs to wears, when it is not 1
266    /// (Djot and Markdown `3.`, HTML `<ol start="3">`). Every item of a list carries it.
267    /// Stored on the `List` entity.
268    pub list_start: Option<i64>,
269    /// Task-list checkbox marker (djot `- [ ]` / `- [x]`). Maps to
270    /// `Block.fmt_marker`. `None` for non-task blocks.
271    pub marker: Option<MarkerType>,
272    pub is_code_block: bool,
273    pub code_language: Option<String>,
274    pub blockquote_depth: u32,
275    /// How many of the quotations the block stands in open at it, the innermost ones:
276    /// quotations as deep as the element's before it but not the same ones. `0` when the
277    /// block goes on in the quotations the element before it stands in, as deep as it
278    /// stands. Two quotations one after the other (Djot and Markdown `> a` and `> b` with a
279    /// blank line between, HTML `<blockquote>` twice) were read as one, since the depth
280    /// alone does not tell them apart.
281    pub blockquote_opens: u32,
282    pub line_height: Option<i64>,
283    pub non_breakable_lines: Option<bool>,
284    /// Start this block on a new page (djot `{page_break_before=true}`). Maps to
285    /// `Block.fmt_page_break_before`. `None` when absent.
286    pub page_break_before: Option<bool>,
287    /// This block opens the document as front matter (djot `{front_matter=true}`). Maps to
288    /// `Block.fmt_front_matter`. `None` when absent.
289    pub front_matter: Option<bool>,
290    pub direction: Option<TextDirection>,
291    pub background_color: Option<String>,
292    /// Paragraph alignment (djot `{alignment=left|right|center|justify}`). Maps
293    /// to `Block.fmt_alignment`. `None` when no alignment attribute is present.
294    pub alignment: Option<Alignment>,
295    /// This block's own space-above (djot `{top_margin=<int>}`). Maps to
296    /// `Block.fmt_top_margin` and overrides the document-wide paragraph
297    /// spacing for this block alone. `None` when absent.
298    pub top_margin: Option<i64>,
299    /// This block's own first-line indent (djot `{text_indent=<int>}`). Maps to
300    /// `Block.fmt_text_indent` and overrides the document-wide first-line
301    /// indent for this block alone. `None` when absent.
302    pub text_indent: Option<i64>,
303    /// The enclosing blockquote's semantic role (djot `{semantic_role=epigraph}`),
304    /// written on the quote's first block because block attributes are the only channel
305    /// djot offers. The importer lifts it onto the `Frame`, where it belongs.
306    pub semantic_role: Option<SemanticRole>,
307}
308
309impl ParsedBlock {
310    /// Returns `true` when this block carries no block-level formatting,
311    /// meaning its content is purely inline.
312    pub fn is_inline_only(&self) -> bool {
313        self.heading_level.is_none()
314            && self.list_style.is_none()
315            && !self.is_code_block
316            && self.blockquote_depth == 0
317            && self.line_height.is_none()
318            && self.non_breakable_lines.is_none()
319            && self.page_break_before.is_none()
320            && self.front_matter.is_none()
321            && self.direction.is_none()
322            && self.background_color.is_none()
323            && self.alignment.is_none()
324            && self.top_margin.is_none()
325            && self.text_indent.is_none()
326    }
327}
328
329// ─── Markdown parsing ────────────────────────────────────────────────
330
331/// Labels referenced as `[^label]` in `markdown` with no `[^label]: ...`
332/// definition anywhere in the document.
333///
334/// pulldown-cmark's default footnote mode (`Options::ENABLE_FOOTNOTES`, i.e.
335/// GitHub's syntax) only emits `Event::FootnoteReference` for a label some
336/// `Tag::FootnoteDefinition` actually defines (verified against 0.13) — an
337/// undefined reference silently decomposes into three ordinary `Text` events
338/// (`"["`, `"^label"`, `"]"`) instead, indistinguishable from a reader having
339/// typed literal brackets. `Options::ENABLE_OLD_FOOTNOTES` recognises it, but
340/// as `parse_markdown`'s doc comment explains, trades away multi-paragraph
341/// definition bodies to get there. Neither flag alone gives both, so
342/// `parse_markdown` hands pulldown-cmark a synthetic empty definition for
343/// every such label instead — real enough for it to recognise the reference,
344/// thrown away again before `elements` is returned.
345///
346/// Finding "every such label" is two parts: which labels pulldown-cmark's own
347/// block scanner (a first pass, at the caller's `options`) already recognises
348/// as truly defined, and which `[^…]`-shaped spans exist in the raw text at
349/// all. The second part is a plain scan, not a parser — it can over-match
350/// (inside a code span, inside a fenced block, a coincidental `[^x]:` in the
351/// middle of a sentence that real block parsing would never treat as a
352/// definition), but over-matching only ever produces an unused synthetic
353/// definition: pulldown-cmark's real, correct parse of the augmented text is
354/// what decides whether any span actually becomes a reference, and
355/// `parse_markdown` drops every synthesized definition regardless of whether
356/// that happened.
357fn dangling_footnote_labels(
358    markdown: &str,
359    options: pulldown_cmark::Options,
360) -> std::collections::BTreeSet<String> {
361    use pulldown_cmark::{Event, Parser, Tag};
362
363    let mut defined: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
364    for event in Parser::new_ext(markdown, options) {
365        if let Event::Start(Tag::FootnoteDefinition(label)) = event {
366            defined.insert(label.to_string());
367        }
368    }
369
370    let mut referenced: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
371    let bytes = markdown.as_bytes();
372    let mut search_from = 0usize;
373    while let Some(rel) = markdown[search_from..].find("[^") {
374        let open = search_from + rel;
375        let label_start = open + 2;
376        let Some(close_rel) = markdown[label_start..].find(']') else {
377            break;
378        };
379        let close = label_start + close_rel;
380        let label = &markdown[label_start..close];
381        // `]:` immediately after is definition syntax, not a reference —
382        // pulldown-cmark's own scan above already supplied the ground truth
383        // for which labels are truly defined; this only needs to avoid
384        // counting a definition's own label as a "reference" candidate.
385        let looks_like_definition = bytes.get(close + 1) == Some(&b':');
386        if !label.is_empty() && !looks_like_definition && !label.chars().any(char::is_whitespace) {
387            referenced.insert(label.to_string());
388        }
389        search_from = close + 1;
390    }
391
392    referenced.difference(&defined).cloned().collect()
393}
394
395/// Read `markdown` into blocks and tables.
396///
397/// The parser reads nested containers without recursing, whatever their depth. The
398/// quotations the text holds are kept within
399/// [`MAX_QUOTE_LEVELS`](crate::parser_tools::quote_depth::MAX_QUOTE_LEVELS) (see
400/// `quote_depth`): the writers walk one call deeper for each, and a text nested 500
401/// quotations deep loaded, then aborted the process at its first save.
402pub fn parse_markdown(markdown: &str) -> Vec<ParsedElement> {
403    use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
404
405    // ENABLE_FOOTNOTES is required for pulldown-cmark to parse `[^label]`
406    // references and `[^label]: body` definitions at all — without it both
407    // fall through as plain link-reference-shaped text. Deliberately NOT
408    // `ENABLE_OLD_FOOTNOTES` too: that variant makes a reference with no
409    // definition survive (see `dangling_footnote_labels`), but in exchange
410    // breaks multi-paragraph definition bodies — a blank line before an
411    // indented continuation escapes the definition and becomes a sibling
412    // code block, which is exactly the shape this crate's own Markdown
413    // exporter writes for a note with more than one paragraph. Getting both
414    // is `dangling_footnote_labels`'s job, not an `Options` flag's.
415    let options = Options::ENABLE_STRIKETHROUGH
416        | Options::ENABLE_TABLES
417        | Options::ENABLE_TASKLISTS
418        | Options::ENABLE_FOOTNOTES;
419
420    // See `dangling_footnote_labels`: pulldown-cmark's GFM-style footnotes
421    // only recognise `[^label]` as a reference when some definition exists
422    // for it anywhere in the document, so an undefined one — the normal
423    // state for a host that owns note bodies itself — has to be given a
424    // throwaway definition before parsing, or it silently becomes the
425    // literal text "[^label]" instead of surviving as a reference.
426    let dangling = dangling_footnote_labels(markdown, options);
427    let augmented_owner;
428    let source: &str = if dangling.is_empty() {
429        markdown
430    } else {
431        augmented_owner = dangling
432            .iter()
433            .fold(markdown.to_string(), |mut acc, label| {
434                acc.push_str("\n\n[^");
435                acc.push_str(label);
436                acc.push_str("]:\n");
437                acc
438            });
439        &augmented_owner
440    };
441    let parser = Parser::new_ext(source, options);
442
443    let mut elements: Vec<ParsedElement> = Vec::new();
444    let mut current_spans: Vec<ParsedSpan> = Vec::new();
445    let mut current_heading: Option<i64> = None;
446    let mut current_list_style: Option<ListStyle> = None;
447    let mut current_list_start: Option<i64> = None;
448    let mut is_code_block = false;
449    let mut code_language: Option<String> = None;
450    let mut blockquote_depth: u32 = 0;
451    // Where `elements` stood as each open quotation began (see `mark_quotation_opened`).
452    let mut quotations_open: Vec<usize> = Vec::new();
453    let mut in_block = false;
454
455    // Formatting state stack
456    let mut bold = false;
457    let mut italic = false;
458    let mut strikeout = false;
459    let mut link_href: Option<String> = None;
460    // Set between an image's Start and End; its alt text arrives as Text events.
461    let mut pending_image: Option<ParsedImage> = None;
462
463    // The label and element index of the footnote definition currently open,
464    // if any. Mirrors `parse_djot`'s `footnote_open`: a definition's body is
465    // ordinary block content, parsed by the same machinery as everything
466    // else, then lifted back out at `End` into its own top-level element so
467    // it never becomes part of the flow it was written in. CommonMark
468    // footnote definitions cannot nest, so one slot suffices.
469    let mut footnote_open: Option<(String, usize)> = None;
470
471    // List style stack for nested lists (also tracks nesting depth), and the start of each
472    // of those lists beside it.
473    let mut list_stack: Vec<Option<ListStyle>> = Vec::new();
474    let mut list_start_stack: Vec<Option<i64>> = Vec::new();
475    let mut current_list_indent: u32 = 0;
476
477    // Table tracking state
478    let mut in_table = false;
479    let mut in_table_head = false;
480    let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
481    let mut current_row_cells: Vec<ParsedTableCell> = Vec::new();
482    let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
483    let mut table_header_rows: usize = 0;
484
485    for event in parser {
486        match event {
487            Event::Start(Tag::Paragraph) => {
488                in_block = true;
489                current_heading = None;
490                is_code_block = false;
491            }
492            Event::End(TagEnd::Paragraph) => {
493                if !current_spans.is_empty() || in_block {
494                    elements.push(ParsedElement::Block(ParsedBlock {
495                        spans: std::mem::take(&mut current_spans),
496                        heading_level: current_heading.take(),
497                        list_style: current_list_style.clone(),
498                        list_indent: current_list_indent,
499                        list_prefix: String::new(),
500                        list_suffix: String::new(),
501                        list_start: current_list_start,
502                        marker: None,
503                        is_code_block: false,
504                        code_language: None,
505                        blockquote_depth,
506                        blockquote_opens: 0,
507                        line_height: None,
508                        non_breakable_lines: None,
509                        page_break_before: None,
510                        front_matter: None,
511                        direction: None,
512                        background_color: None,
513                        alignment: None,
514                        top_margin: None,
515                        text_indent: None,
516                        semantic_role: None,
517                    }));
518                }
519                in_block = false;
520                current_list_style = None;
521                current_list_start = None;
522            }
523            Event::Start(Tag::Heading { level, .. }) => {
524                in_block = true;
525                current_heading = Some(heading_level_to_i64(level));
526                is_code_block = false;
527            }
528            Event::End(TagEnd::Heading(_)) => {
529                elements.push(ParsedElement::Block(ParsedBlock {
530                    spans: std::mem::take(&mut current_spans),
531                    heading_level: current_heading.take(),
532                    list_style: None,
533                    list_indent: 0,
534                    list_prefix: String::new(),
535                    list_suffix: String::new(),
536                    list_start: None,
537                    marker: None,
538                    is_code_block: false,
539                    code_language: None,
540                    blockquote_depth,
541                    blockquote_opens: 0,
542                    line_height: None,
543                    non_breakable_lines: None,
544                    page_break_before: None,
545                    front_matter: None,
546                    direction: None,
547                    background_color: None,
548                    alignment: None,
549                    top_margin: None,
550                    text_indent: None,
551                    semantic_role: None,
552                }));
553                in_block = false;
554            }
555            Event::Start(Tag::List(ordered)) => {
556                let style = if ordered.is_some() {
557                    Some(ListStyle::Decimal)
558                } else {
559                    Some(ListStyle::Disc)
560                };
561                list_stack.push(style);
562                list_start_stack.push(ordered.and_then(list_start_of));
563            }
564            Event::End(TagEnd::List(_)) => {
565                list_stack.pop();
566                list_start_stack.pop();
567            }
568            Event::Start(Tag::Item) => {
569                // Flush any accumulated spans from the parent item before
570                // starting a child item in a tight list
571                if !current_spans.is_empty() {
572                    elements.push(ParsedElement::Block(ParsedBlock {
573                        spans: std::mem::take(&mut current_spans),
574                        heading_level: None,
575                        list_style: current_list_style.clone(),
576                        list_indent: current_list_indent,
577                        list_prefix: String::new(),
578                        list_suffix: String::new(),
579                        list_start: current_list_start,
580                        marker: None,
581                        is_code_block: false,
582                        code_language: None,
583                        blockquote_depth,
584                        blockquote_opens: 0,
585                        line_height: None,
586                        non_breakable_lines: None,
587                        page_break_before: None,
588                        front_matter: None,
589                        direction: None,
590                        background_color: None,
591                        alignment: None,
592                        top_margin: None,
593                        text_indent: None,
594                        semantic_role: None,
595                    }));
596                }
597                in_block = true;
598                current_list_style = list_stack.last().cloned().flatten();
599                current_list_start = list_start_stack.last().copied().flatten();
600                current_list_indent = if list_stack.is_empty() {
601                    0
602                } else {
603                    (list_stack.len() - 1) as u32
604                };
605            }
606            Event::End(TagEnd::Item) => {
607                // The paragraph inside the item will have already been flushed,
608                // but if there was no inner paragraph (tight list), flush now.
609                if !current_spans.is_empty() {
610                    elements.push(ParsedElement::Block(ParsedBlock {
611                        spans: std::mem::take(&mut current_spans),
612                        heading_level: None,
613                        list_style: current_list_style.clone(),
614                        list_indent: current_list_indent,
615                        list_prefix: String::new(),
616                        list_suffix: String::new(),
617                        list_start: current_list_start,
618                        marker: None,
619                        is_code_block: false,
620                        code_language: None,
621                        blockquote_depth,
622                        blockquote_opens: 0,
623                        line_height: None,
624                        non_breakable_lines: None,
625                        page_break_before: None,
626                        front_matter: None,
627                        direction: None,
628                        background_color: None,
629                        alignment: None,
630                        top_margin: None,
631                        text_indent: None,
632                        semantic_role: None,
633                    }));
634                }
635                in_block = false;
636                current_list_style = None;
637                current_list_start = None;
638            }
639            Event::Start(Tag::CodeBlock(kind)) => {
640                in_block = true;
641                is_code_block = true;
642                code_language = match &kind {
643                    pulldown_cmark::CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
644                        Some(lang.to_string())
645                    }
646                    _ => None,
647                };
648            }
649            Event::End(TagEnd::CodeBlock) => {
650                // pulldown-cmark appends a trailing '\n' to code block text — strip it
651                if let Some(last) = current_spans.last_mut()
652                    && last.text.ends_with('\n')
653                {
654                    last.text.truncate(last.text.len() - 1);
655                }
656                elements.push(ParsedElement::Block(ParsedBlock {
657                    spans: std::mem::take(&mut current_spans),
658                    heading_level: None,
659                    list_style: None,
660                    list_indent: 0,
661                    list_prefix: String::new(),
662                    list_suffix: String::new(),
663                    list_start: None,
664                    marker: None,
665                    is_code_block: true,
666                    code_language: code_language.take(),
667                    blockquote_depth,
668                    blockquote_opens: 0,
669                    line_height: None,
670                    non_breakable_lines: None,
671                    page_break_before: None,
672                    front_matter: None,
673                    direction: None,
674                    background_color: None,
675                    alignment: None,
676                    top_margin: None,
677                    text_indent: None,
678                    semantic_role: None,
679                }));
680                in_block = false;
681                is_code_block = false;
682            }
683            // ─── Table events ───────────────────────────────────────
684            Event::Start(Tag::Table(_)) => {
685                in_table = true;
686                in_table_head = false;
687                table_rows.clear();
688                current_row_cells.clear();
689                current_cell_spans.clear();
690                table_header_rows = 0;
691            }
692            Event::End(TagEnd::Table) => {
693                let own_cells = cells_in_rows(&table_rows);
694                push_table(
695                    &mut elements,
696                    ParsedTable {
697                        header_rows: table_header_rows,
698                        rows: std::mem::take(&mut table_rows),
699                        blockquote_depth,
700                        blockquote_opens: 0,
701                    },
702                    own_cells,
703                );
704                in_table = false;
705            }
706            Event::Start(Tag::TableHead) => {
707                in_table_head = true;
708                current_row_cells.clear();
709            }
710            Event::End(TagEnd::TableHead) => {
711                // Flush the header row
712                table_rows.push(std::mem::take(&mut current_row_cells));
713                table_header_rows += 1;
714                in_table_head = false;
715            }
716            Event::Start(Tag::TableRow) => {
717                current_row_cells.clear();
718            }
719            Event::End(TagEnd::TableRow) if !in_table_head => {
720                // Body rows only — header row is flushed in End(TableHead)
721                table_rows.push(std::mem::take(&mut current_row_cells));
722            }
723            Event::Start(Tag::TableCell) => {
724                current_cell_spans.clear();
725            }
726            Event::End(TagEnd::TableCell) => {
727                current_row_cells.push(ParsedTableCell {
728                    spans: std::mem::take(&mut current_cell_spans),
729                });
730            }
731            // ─── Inline formatting ──────────────────────────────────
732            Event::Start(Tag::Emphasis) => {
733                italic = true;
734            }
735            Event::End(TagEnd::Emphasis) => {
736                italic = false;
737            }
738            Event::Start(Tag::Strong) => {
739                bold = true;
740            }
741            Event::End(TagEnd::Strong) => {
742                bold = false;
743            }
744            Event::Start(Tag::Strikethrough) => {
745                strikeout = true;
746            }
747            Event::End(TagEnd::Strikethrough) => {
748                strikeout = false;
749            }
750            Event::Start(Tag::Link { dest_url, .. }) => {
751                link_href = Some(dest_url.to_string());
752            }
753            Event::End(TagEnd::Link) => {
754                link_href = None;
755            }
756            // Markdown has no syntax for display size, so width/height stay 0
757            // and the caller supplies the image's intrinsic dimensions.
758            Event::Start(Tag::Image { dest_url, .. }) => {
759                pending_image = Some(ParsedImage {
760                    src: dest_url.to_string(),
761                    alt: String::new(),
762                    width: 0,
763                    height: 0,
764                });
765            }
766            Event::End(TagEnd::Image) => {
767                if let Some(image) = pending_image.take() {
768                    let span = ParsedSpan {
769                        text: String::new(),
770                        bold,
771                        italic,
772                        underline: false,
773                        strikeout,
774                        code: false,
775                        superscript: false,
776                        subscript: false,
777                        link_href: link_href.clone(),
778                        image: Some(image),
779                        footnote_ref: None,
780                    };
781                    if in_table {
782                        current_cell_spans.push(span);
783                    } else {
784                        if !in_block {
785                            in_block = true;
786                        }
787                        current_spans.push(span);
788                    }
789                }
790            }
791            Event::Text(text) => {
792                // Inside an image this is its alt text, which pulldown-cmark
793                // emits as an ordinary Text event. Without this guard it fell
794                // through and landed in the paragraph as prose — the image was
795                // dropped and its description silently became manuscript text.
796                if let Some(img) = pending_image.as_mut() {
797                    img.alt.push_str(&text);
798                    continue;
799                }
800                let span = ParsedSpan {
801                    text: text.to_string(),
802                    bold,
803                    italic,
804                    underline: false,
805                    strikeout,
806                    code: is_code_block,
807                    superscript: false,
808                    subscript: false,
809                    link_href: link_href.clone(),
810                    image: None,
811                    footnote_ref: None,
812                };
813                if in_table {
814                    current_cell_spans.push(span);
815                } else {
816                    if !in_block {
817                        in_block = true;
818                    }
819                    current_spans.push(span);
820                }
821            }
822            Event::Code(text) => {
823                let span = ParsedSpan {
824                    text: text.to_string(),
825                    bold,
826                    italic,
827                    underline: false,
828                    strikeout,
829                    code: true,
830                    superscript: false,
831                    subscript: false,
832                    link_href: link_href.clone(),
833                    image: None,
834                    footnote_ref: None,
835                };
836                if in_table {
837                    current_cell_spans.push(span);
838                } else {
839                    if !in_block {
840                        in_block = true;
841                    }
842                    current_spans.push(span);
843                }
844            }
845            Event::SoftBreak => {
846                let span = ParsedSpan {
847                    text: " ".to_string(),
848                    bold,
849                    italic,
850                    underline: false,
851                    strikeout,
852                    code: false,
853                    superscript: false,
854                    subscript: false,
855                    link_href: link_href.clone(),
856                    image: None,
857                    footnote_ref: None,
858                };
859                if in_table {
860                    current_cell_spans.push(span);
861                } else {
862                    current_spans.push(span);
863                }
864            }
865            // A hard break (two spaces or a backslash at the end of the line) is a line
866            // of the same block, as in the Djot reader. A table cell is one line, and so
867            // is an image's description.
868            Event::HardBreak if pending_image.is_some() => {
869                if let Some(img) = pending_image.as_mut() {
870                    img.alt.push(' ');
871                }
872            }
873            Event::HardBreak => {
874                let span = ParsedSpan {
875                    text: if in_table { " " } else { LINE_SEPARATOR_STR }.to_string(),
876                    bold,
877                    italic,
878                    underline: false,
879                    strikeout,
880                    code: false,
881                    superscript: false,
882                    subscript: false,
883                    link_href: link_href.clone(),
884                    image: None,
885                    footnote_ref: None,
886                };
887                if in_table {
888                    current_cell_spans.push(span);
889                } else {
890                    current_spans.push(span);
891                }
892            }
893            Event::Start(Tag::BlockQuote(_)) => {
894                blockquote_depth += 1;
895                quotations_open.push(elements.len());
896            }
897            Event::End(TagEnd::BlockQuote(_)) => {
898                blockquote_depth = blockquote_depth.saturating_sub(1);
899                if let Some(first) = quotations_open.pop() {
900                    mark_quotation_opened(&mut elements, first);
901                }
902            }
903            // ── Footnote definitions ──
904            //
905            // `[^label]: body` opens a block-level container; its body is
906            // ordinary paragraphs/lists/etc., pushed onto `elements` by the
907            // ordinary machinery above. `End` drains everything pushed since
908            // `Start` back out into one `FootnoteDefinition`, exactly as
909            // `parse_djot` does for the same `[^label]: body` syntax.
910            // Flushing any spans left open by an interrupted block first
911            // matches every other container boundary in this parser
912            // (List/Item/BlockQuote).
913            Event::Start(Tag::FootnoteDefinition(label)) => {
914                if !current_spans.is_empty() {
915                    elements.push(ParsedElement::Block(ParsedBlock {
916                        spans: std::mem::take(&mut current_spans),
917                        heading_level: current_heading.take(),
918                        list_style: current_list_style.clone(),
919                        list_indent: current_list_indent,
920                        list_prefix: String::new(),
921                        list_suffix: String::new(),
922                        list_start: current_list_start,
923                        marker: None,
924                        is_code_block: false,
925                        code_language: None,
926                        blockquote_depth,
927                        blockquote_opens: 0,
928                        line_height: None,
929                        non_breakable_lines: None,
930                        page_break_before: None,
931                        front_matter: None,
932                        direction: None,
933                        background_color: None,
934                        alignment: None,
935                        top_margin: None,
936                        text_indent: None,
937                        semantic_role: None,
938                    }));
939                }
940                footnote_open = Some((label.to_string(), elements.len()));
941            }
942            Event::End(TagEnd::FootnoteDefinition) => {
943                if !current_spans.is_empty() {
944                    elements.push(ParsedElement::Block(ParsedBlock {
945                        spans: std::mem::take(&mut current_spans),
946                        heading_level: current_heading.take(),
947                        list_style: current_list_style.clone(),
948                        list_indent: current_list_indent,
949                        list_prefix: String::new(),
950                        list_suffix: String::new(),
951                        list_start: current_list_start,
952                        marker: None,
953                        is_code_block: false,
954                        code_language: None,
955                        blockquote_depth,
956                        blockquote_opens: 0,
957                        line_height: None,
958                        non_breakable_lines: None,
959                        page_break_before: None,
960                        front_matter: None,
961                        direction: None,
962                        background_color: None,
963                        alignment: None,
964                        top_margin: None,
965                        text_indent: None,
966                        semantic_role: None,
967                    }));
968                }
969                if let Some((label, start)) = footnote_open.take() {
970                    let blocks: Vec<ParsedBlock> = elements
971                        .drain(start..)
972                        .filter_map(|e| match e {
973                            ParsedElement::Block(b) => Some(b),
974                            // A table inside a footnote is not representable
975                            // as note content, same limitation as djot.
976                            _ => None,
977                        })
978                        .collect();
979                    elements.push(ParsedElement::FootnoteDefinition { label, blocks });
980                }
981            }
982            // A footnote reference. pulldown-cmark emits this whether or not
983            // a matching `[^label]:` definition exists anywhere in the
984            // document — a dangling reference (the normal state for a host
985            // that owns note bodies itself) must survive just the same,
986            // mirroring `parse_djot`'s `E::FootnoteReference` handling.
987            Event::FootnoteReference(label) => {
988                let span = ParsedSpan {
989                    text: String::new(),
990                    bold,
991                    italic,
992                    underline: false,
993                    strikeout,
994                    code: false,
995                    superscript: false,
996                    subscript: false,
997                    link_href: link_href.clone(),
998                    image: None,
999                    footnote_ref: Some(label.to_string()),
1000                };
1001                if in_table {
1002                    current_cell_spans.push(span);
1003                } else {
1004                    if !in_block {
1005                        in_block = true;
1006                    }
1007                    current_spans.push(span);
1008                }
1009            }
1010            _ => {}
1011        }
1012    }
1013
1014    // Flush any remaining content
1015    if !current_spans.is_empty() {
1016        elements.push(ParsedElement::Block(ParsedBlock {
1017            spans: std::mem::take(&mut current_spans),
1018            heading_level: current_heading,
1019            list_style: current_list_style,
1020            list_indent: current_list_indent,
1021            list_prefix: String::new(),
1022            list_suffix: String::new(),
1023            list_start: current_list_start,
1024            marker: None,
1025            is_code_block,
1026            code_language: code_language.take(),
1027            blockquote_depth,
1028            blockquote_opens: 0,
1029            line_height: None,
1030            non_breakable_lines: None,
1031            page_break_before: None,
1032            front_matter: None,
1033            direction: None,
1034            background_color: None,
1035            alignment: None,
1036            top_margin: None,
1037            text_indent: None,
1038            semantic_role: None,
1039        }));
1040    }
1041
1042    // Drop the throwaway definitions `dangling_footnote_labels` asked for —
1043    // they exist only to make pulldown-cmark recognise the reference as
1044    // real, never to become a note the document owns. `elements` cannot
1045    // become empty from this: a label reached `dangling` only via a
1046    // reference actually present in `markdown`, so its surrounding block
1047    // survives even with the synthetic definition gone.
1048    if !dangling.is_empty() {
1049        elements.retain(
1050            |e| !matches!(e, ParsedElement::FootnoteDefinition { label, .. } if dangling.contains(label)),
1051        );
1052    }
1053
1054    // If no elements were parsed, create a single empty paragraph
1055    if elements.is_empty() {
1056        elements.push(ParsedElement::Block(ParsedBlock {
1057            spans: vec![ParsedSpan {
1058                text: String::new(),
1059                ..Default::default()
1060            }],
1061            heading_level: None,
1062            list_style: None,
1063            list_indent: 0,
1064            list_prefix: String::new(),
1065            list_suffix: String::new(),
1066            list_start: None,
1067            marker: None,
1068            is_code_block: false,
1069            code_language: None,
1070            blockquote_depth: 0,
1071            blockquote_opens: 0,
1072            line_height: None,
1073            non_breakable_lines: None,
1074            page_break_before: None,
1075            front_matter: None,
1076            direction: None,
1077            background_color: None,
1078            alignment: None,
1079            top_margin: None,
1080            text_indent: None,
1081            semantic_role: None,
1082        }));
1083    }
1084
1085    // Quotations past the ceiling a document holds are flattened into the deepest kept
1086    // (see `quote_depth`): the writers walk one call deeper for each.
1087    crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1088    elements
1089}
1090
1091fn heading_level_to_i64(level: pulldown_cmark::HeadingLevel) -> i64 {
1092    use pulldown_cmark::HeadingLevel;
1093    match level {
1094        HeadingLevel::H1 => 1,
1095        HeadingLevel::H2 => 2,
1096        HeadingLevel::H3 => 3,
1097        HeadingLevel::H4 => 4,
1098        HeadingLevel::H5 => 5,
1099        HeadingLevel::H6 => 6,
1100    }
1101}
1102
1103// ─── HTML parsing ────────────────────────────────────────────────────
1104
1105use scraper::Node;
1106
1107/// Parsed CSS block-level styles from an inline `style` attribute.
1108#[derive(Debug, Clone, Default)]
1109struct BlockStyles {
1110    line_height: Option<i64>,
1111    non_breakable_lines: Option<bool>,
1112    page_break_before: Option<bool>,
1113    direction: Option<TextDirection>,
1114    background_color: Option<String>,
1115    /// `white-space` asks for the source's own spacing to be kept verbatim.
1116    /// Separate from `non_breakable_lines` because the two do not coincide:
1117    /// `nowrap` forbids wrapping but still collapses runs of spaces, so a
1118    /// block carrying it must be normalised like any other.
1119    preserve_whitespace: Option<bool>,
1120}
1121
1122/// Parse relevant CSS properties from an inline style string.
1123/// Handles: line-height, white-space, break-before/page-break-before, direction,
1124/// background-color.
1125fn parse_block_styles(style: &str) -> BlockStyles {
1126    let mut result = BlockStyles::default();
1127    for part in style.split(';') {
1128        let part = part.trim();
1129        if let Some((prop, val)) = part.split_once(':') {
1130            let prop = prop.trim().to_ascii_lowercase();
1131            let val = val.trim();
1132            match prop.as_str() {
1133                "line-height" => {
1134                    // Try parsing as a plain number (multiplier)
1135                    if let Ok(v) = val.parse::<f64>() {
1136                        result.line_height = Some((v * 1000.0) as i64);
1137                    }
1138                }
1139                "white-space" if val == "pre" || val == "nowrap" || val == "pre-wrap" => {
1140                    result.non_breakable_lines = Some(true);
1141                    if val != "nowrap" {
1142                        result.preserve_whitespace = Some(true);
1143                    }
1144                }
1145                // CSS3 `break-before` and its CSS2 predecessor `page-break-before` mean
1146                // the same thing; both are read because both are written (browsers still
1147                // want the legacy spelling, and so do most EPUB engines).
1148                "break-before" | "page-break-before" => {
1149                    result.page_break_before = match val.to_ascii_lowercase().as_str() {
1150                        "page" | "always" | "left" | "right" | "recto" | "verso" => Some(true),
1151                        "avoid" | "auto" => Some(false),
1152                        _ => None,
1153                    };
1154                }
1155                "direction" => {
1156                    if val.eq_ignore_ascii_case("rtl") {
1157                        result.direction = Some(TextDirection::RightToLeft);
1158                    } else if val.eq_ignore_ascii_case("ltr") {
1159                        result.direction = Some(TextDirection::LeftToRight);
1160                    }
1161                }
1162                "background-color" | "background" => {
1163                    result.background_color = Some(val.to_string());
1164                }
1165                _ => {}
1166            }
1167        }
1168    }
1169    result
1170}
1171
1172/// The five characters HTML calls whitespace.
1173///
1174/// Deliberately not `char::is_whitespace`, which is `true` for U+00A0: a
1175/// no-break space is a character the writer typed — the one French typography
1176/// puts before a colon, holding "Attention&nbsp;:" together — and collapsing it
1177/// away is a silent edit of the text, not of its layout.
1178fn is_html_space(ch: char) -> bool {
1179    matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{0C}')
1180}
1181
1182/// Whether a span run holds anything a reader would see.
1183fn spans_carry_content(spans: &[ParsedSpan]) -> bool {
1184    spans
1185        .iter()
1186        .any(|s| !s.text.is_empty() || s.image.is_some() || s.footnote_ref.is_some())
1187}
1188
1189/// Apply the CSS `white-space: normal` rules to one block's inline spans: every
1190/// run of HTML whitespace collapses to a single space, and the runs at the
1191/// block's two ends disappear entirely.
1192///
1193/// Per block rather than per text node, because a run crosses span boundaries —
1194/// `<b>word</b>\n<i>next</i>` is one collapsible run split over three nodes —
1195/// and because only the block knows where its own ends are.
1196///
1197/// Without this, an exporter's own source formatting reaches the document as
1198/// text. LibreOffice writes a newline after every `<p …>` and hard-wraps the
1199/// prose inside it at about seventy columns, so pasting a whole book put a
1200/// blank line before every paragraph and a break through the middle of every
1201/// sentence.
1202fn collapse_inline_whitespace(spans: &mut [ParsedSpan]) {
1203    fn is_replaced(span: &ParsedSpan) -> bool {
1204        span.image.is_some() || span.footnote_ref.is_some()
1205    }
1206
1207    // Collapse within each span first. Nothing non-empty is emptied here, which
1208    // is what lets the second pass read `is_empty()` as "this span held no text
1209    // to begin with" — the marker a `<br>` leaves behind.
1210    for span in spans.iter_mut() {
1211        if is_replaced(span) {
1212            continue;
1213        }
1214        let mut out = String::with_capacity(span.text.len());
1215        let mut in_run = false;
1216        for ch in span.text.chars() {
1217            if is_html_space(ch) {
1218                if !in_run {
1219                    out.push(' ');
1220                    in_run = true;
1221                }
1222            } else {
1223                out.push(ch);
1224                in_run = false;
1225            }
1226        }
1227        span.text = out;
1228    }
1229
1230    // The block's start counts as "already ended on a space", so a run opening
1231    // the block is dropped outright. A line break (the span a `<br>` leaves) ends one
1232    // line and starts the next, so the runs either side of it go as the block's own
1233    // ends do: `x <br> y` is two lines, "x" and "y".
1234    let mut prev_ends_with_space = true;
1235    // The last span holding text since the line began, whose trailing space a line
1236    // break takes off.
1237    let mut line_end: Option<usize> = None;
1238    for i in 0..spans.len() {
1239        let span = &mut spans[i];
1240        if is_replaced(span) {
1241            prev_ends_with_space = false;
1242            line_end = None;
1243            continue;
1244        }
1245        if span.text.is_empty() {
1246            // A span that held no text to begin with. It contributes none, but it
1247            // still ends the collapsible run either side of it.
1248            prev_ends_with_space = false;
1249            continue;
1250        }
1251        if span.text == LINE_SEPARATOR_STR {
1252            if let Some(end) = line_end.take()
1253                && spans[end].text.ends_with(' ')
1254            {
1255                spans[end].text.pop();
1256            }
1257            prev_ends_with_space = true;
1258            continue;
1259        }
1260        if prev_ends_with_space && span.text.starts_with(' ') {
1261            span.text.remove(0);
1262        }
1263        // A span emptied by that drop leaves the run open rather than closing
1264        // it: the space it carried belongs to the run its neighbours share.
1265        if !span.text.is_empty() {
1266            prev_ends_with_space = span.text.ends_with(' ');
1267            line_end = Some(i);
1268        }
1269    }
1270
1271    // The block's trailing run, in whichever span still holds it. Replaced
1272    // content stops the walk: a space before a trailing image is between two
1273    // pieces of content, not after the last one. A line break at the very end
1274    // starts no line of its own, as a browser shows it (a second one does: `a<br><br>`
1275    // is "a" and an empty line), so the last one goes, with the run before it.
1276    let mut trailing_break_dropped = false;
1277    for span in spans.iter_mut().rev() {
1278        if is_replaced(span) {
1279            break;
1280        }
1281        if span.text.is_empty() {
1282            continue;
1283        }
1284        if span.text == LINE_SEPARATOR_STR {
1285            if trailing_break_dropped {
1286                break;
1287            }
1288            span.text.clear();
1289            trailing_break_dropped = true;
1290            continue;
1291        }
1292        if span.text.ends_with(' ') {
1293            span.text.pop();
1294        }
1295        break;
1296    }
1297}
1298
1299/// `block` as one block per line of its text, a line ending at each `\n`.
1300///
1301/// A `\n` is not a line break inside a block, which the model holds as
1302/// [`LINE_SEPARATOR`]: in the rope it separates two blocks, so a new line of this kind
1303/// is a new block, which is what a plain-text paste makes of one. Text that kept its
1304/// source's line breaks (`<pre>`, `white-space: pre`) held them inside one block, where
1305/// the editor showed them as lines and the Djot writer had nothing to write them as.
1306/// Written raw, a blank line ended the paragraph and the lines after it were read back
1307/// as the lists, headings and quotations their markers spelled, however deep.
1308///
1309/// Each line keeps the block's own formatting and its spans' formatting, but for the
1310/// page break, which stays with the first line: it is where the block starts, and on
1311/// every line it put each on a page of its own. An empty line is an empty block. The
1312/// line break that ends the last line starts no line of its own, as a browser shows it.
1313/// A block without a line break is returned as it is.
1314///
1315/// Every line is built from copies of the block and of the span it comes from that
1316/// hold no text, so the split costs in proportion to the text: built from whole copies,
1317/// each line copied the whole text again, and a paste of 64,000 preformatted lines
1318/// took seconds.
1319pub fn split_block_at_line_breaks(mut block: ParsedBlock) -> Vec<ParsedBlock> {
1320    let breaks = |span: &ParsedSpan| span.image.is_none() && span.text.contains('\n');
1321    if !block.spans.iter().any(breaks) {
1322        return vec![block];
1323    }
1324    let mut lines: Vec<Vec<ParsedSpan>> = vec![Vec::new()];
1325    for mut span in std::mem::take(&mut block.spans) {
1326        if !breaks(&span) {
1327            push_to_last_line(&mut lines, span);
1328            continue;
1329        }
1330        let text = std::mem::take(&mut span.text);
1331        for (i, piece) in text.split('\n').enumerate() {
1332            if i > 0 {
1333                lines.push(Vec::new());
1334            }
1335            if !piece.is_empty() {
1336                push_to_last_line(
1337                    &mut lines,
1338                    ParsedSpan {
1339                        text: piece.to_string(),
1340                        ..span.clone()
1341                    },
1342                );
1343            }
1344        }
1345    }
1346    if lines.len() > 1 && lines.last().is_some_and(Vec::is_empty) {
1347        lines.pop();
1348    }
1349    let later_line = ParsedBlock {
1350        page_break_before: None,
1351        blockquote_opens: 0,
1352        ..block.clone()
1353    };
1354    let mut first_line = Some(block);
1355    lines
1356        .into_iter()
1357        .map(|spans| {
1358            let spans = if spans.is_empty() {
1359                vec![ParsedSpan::default()]
1360            } else {
1361                spans
1362            };
1363            match first_line.take() {
1364                Some(first) => ParsedBlock { spans, ..first },
1365                None => ParsedBlock {
1366                    spans,
1367                    ..later_line.clone()
1368                },
1369            }
1370        })
1371        .collect()
1372}
1373
1374/// Append `span` to the last of `lines`, which [`split_block_at_line_breaks`] keeps
1375/// non-empty.
1376fn push_to_last_line(lines: &mut Vec<Vec<ParsedSpan>>, span: ParsedSpan) {
1377    match lines.last_mut() {
1378        Some(line) => line.push(span),
1379        None => lines.push(vec![span]),
1380    }
1381}
1382
1383pub fn parse_html(html: &str) -> Vec<ParsedBlock> {
1384    ParsedElement::flatten_to_blocks(parse_html_elements(html))
1385}
1386
1387/// The attribute an HTML producer uses to say "this element *is* a footnote
1388/// reference", and the label it names.
1389///
1390/// A data attribute on an otherwise-ordinary element, rather than a literal
1391/// `[^label]` in the text, because the text a producer emits is *escaped* on the
1392/// way to Djot — [`escape_djot_inline`](crate::parser_tools::djot_escape::escape_djot_inline)
1393/// neutralises `[`, `]` and `^`, so a reference smuggled through as characters
1394/// arrives as `\[\^1\]` and is prose rather than a reference. It has to be a
1395/// node before it is serialised, which is what this makes it.
1396///
1397/// [`ParsedSpan::footnote_ref`] is the same field the Markdown/Djot reader
1398/// already fills, so everything downstream — the document model, the Djot
1399/// writer, numbering — needs no change at all.
1400pub const HTML_FOOTNOTE_ATTR: &str = "data-footnote-ref";
1401
1402/// Build a footnote-reference span from any element carrying
1403/// [`HTML_FOOTNOTE_ATTR`].
1404fn html_footnote_span(
1405    el: &scraper::node::Element,
1406    link_href: Option<String>,
1407) -> Option<ParsedSpan> {
1408    let label = el.attr(HTML_FOOTNOTE_ATTR)?.trim();
1409    if label.is_empty() {
1410        return None;
1411    }
1412    Some(ParsedSpan {
1413        text: String::new(),
1414        link_href,
1415        footnote_ref: Some(label.to_string()),
1416        ..Default::default()
1417    })
1418}
1419
1420/// Build an inline-image span from an `<img>` element, if it has a usable
1421/// source.
1422///
1423/// `<img>` was matched by none of the HTML walker's three tag dispatches, so it
1424/// fell into their wildcard arms and was dropped whole — silently, and unlike
1425/// Markdown not even leaving its alt text behind. That is the path a browser or
1426/// Word paste travels.
1427fn html_img_span(el: &scraper::node::Element, link_href: Option<String>) -> Option<ParsedSpan> {
1428    let src = el.attr("src")?;
1429    if src.is_empty() {
1430        return None;
1431    }
1432    let dim = |name: &str| -> i64 {
1433        el.attr(name)
1434            .and_then(|v| v.trim().trim_end_matches("px").parse::<i64>().ok())
1435            .filter(|n| *n > 0)
1436            .unwrap_or(0)
1437    };
1438    Some(ParsedSpan {
1439        text: String::new(),
1440        link_href,
1441        image: Some(ParsedImage {
1442            src: src.to_string(),
1443            alt: el.attr("alt").unwrap_or_default().to_string(),
1444            width: dim("width"),
1445            height: dim("height"),
1446        }),
1447        ..Default::default()
1448    })
1449}
1450
1451/// Whether `node` is a `<ul>` or an `<ol>`.
1452fn is_html_list(node: &Node) -> bool {
1453    node.as_element()
1454        .is_some_and(|el| matches!(el.name(), "ul" | "ol"))
1455}
1456
1457/// Move every list item nested deeper than
1458/// [`MAX_LIST_LEVELS`](crate::parser_tools::list_depth::MAX_LIST_LEVELS)
1459/// lists into the deepest list kept, right after the item holding it, in
1460/// document order: a list nested past that depth comes out flattened to it,
1461/// every item kept, in its order.
1462///
1463/// The reader recurses a few elements deep per list level and stops at
1464/// `MAX_RECURSION_DEPTH`, so a list nested past some eighty levels lost every
1465/// item below that, pasted or loaded. The editing gestures nest no deeper than
1466/// this depth, and an insertion brings any list back to it (see `list_depth`),
1467/// so HTML holding more is read at that depth to begin with. The tree is
1468/// changed in place, without recursion, before it is read.
1469fn flatten_deep_html_lists(tree: &mut ego_tree::Tree<Node>) {
1470    use crate::parser_tools::list_depth::MAX_LIST_LEVELS;
1471    use ego_tree::iter::Edge;
1472
1473    // The lists one level past the deepest kept, in document order.
1474    let mut too_deep: Vec<ego_tree::NodeId> = Vec::new();
1475    let mut depth = 0usize;
1476    for edge in tree.root().traverse() {
1477        match edge {
1478            Edge::Open(node) if is_html_list(node.value()) => {
1479                if depth == MAX_LIST_LEVELS {
1480                    too_deep.push(node.id());
1481                }
1482                depth += 1;
1483            }
1484            Edge::Close(node) if is_html_list(node.value()) => {
1485                depth = depth.saturating_sub(1);
1486            }
1487            _ => {}
1488        }
1489    }
1490
1491    for list in too_deep {
1492        // The child of the deepest kept list that holds `list` (its `<li>`, or
1493        // `list` itself when a list sits right inside another): the items go
1494        // after it.
1495        let Some(list_ref) = tree.get(list) else {
1496            continue;
1497        };
1498        let mut holder = None;
1499        let mut current = list_ref;
1500        while let Some(parent) = current.parent() {
1501            if is_html_list(parent.value()) {
1502                holder = Some(current.id());
1503                break;
1504            }
1505            current = parent;
1506        }
1507        let Some(holder) = holder else {
1508            continue;
1509        };
1510        let mut after = holder;
1511        let items: Vec<ego_tree::NodeId> = list_ref
1512            .descendants()
1513            .filter(|node| {
1514                node.value()
1515                    .as_element()
1516                    .is_some_and(|el| el.name() == "li")
1517            })
1518            .map(|node| node.id())
1519            .collect();
1520        for item in items {
1521            // Each node placed here sits in the deepest kept list, so it has a
1522            // parent to be placed beside.
1523            if tree.get(after).and_then(|node| node.parent()).is_none() {
1524                break;
1525            }
1526            if let Some(mut node) = tree.get_mut(after) {
1527                node.insert_id_after(item);
1528                after = item;
1529            }
1530        }
1531        // What is left of the list is its items' wrappers, or text standing in
1532        // it outside any item. It goes after the items when it holds any text
1533        // (it is already beside them when it sits right in the deepest kept
1534        // list), and away when it holds none.
1535        let holds_text = tree.get(list).is_some_and(|node| {
1536            node.descendants()
1537                .any(|d| d.value().as_text().is_some_and(|t| !t.trim().is_empty()))
1538        });
1539        if !holds_text {
1540            if let Some(mut node) = tree.get_mut(list) {
1541                node.detach();
1542            }
1543        } else if holder != list
1544            && tree.get(after).and_then(|node| node.parent()).is_some()
1545            && let Some(mut node) = tree.get_mut(after)
1546        {
1547            node.insert_id_after(list);
1548        }
1549    }
1550}
1551
1552/// How deep a walk of an HTML tree may go on the caller's own stack: a debug build spends
1553/// between 5 and 8 KiB of stack on each element level it follows, so this is at most some
1554/// 256 KiB, what any thread a host reads on has to spare.
1555const HTML_DEPTH_ON_THE_CALLERS_STACK: usize = 32;
1556
1557/// The deepest an HTML reader follows elements: a subtree nested deeper is read as its
1558/// text, without recursing (see `read_html_tree`).
1559const MAX_HTML_READ_DEPTH: usize = 256;
1560
1561/// The stack a reader of a deeper tree gets on a thread of its own, before what each level
1562/// adds: a spawned thread's.
1563const HTML_READER_STACK: usize = 2 << 20;
1564
1565/// The stack each element level of a deeper tree adds for its reader: two to three times
1566/// what a debug build spends on one.
1567const HTML_STACK_PER_LEVEL: usize = 16 << 10;
1568
1569/// `value` read as HTML reads an integer attribute such as `<ol start>`: after any leading
1570/// white space, an optional sign and the digits up to the first character that is not one.
1571/// `None` when no digit follows. A number past what 64 bits hold reads as the largest they
1572/// hold on its side of 0. A browser numbers `<ol start="5x">` from 5, where a strict parse
1573/// of the whole value numbered it from 1.
1574fn html_integer(value: &str) -> Option<i64> {
1575    let value = value.trim_start_matches(|c: char| c.is_ascii_whitespace());
1576    let (negative, digits) = match value.as_bytes().first() {
1577        Some(b'-') => (true, &value[1..]),
1578        Some(b'+') => (false, &value[1..]),
1579        _ => (false, value),
1580    };
1581    let digits: &str = digits
1582        .split(|c: char| !c.is_ascii_digit())
1583        .next()
1584        .unwrap_or("");
1585    if digits.is_empty() {
1586        return None;
1587    }
1588    let magnitude = digits.bytes().fold(0i64, |number, digit| {
1589        number
1590            .saturating_mul(10)
1591            .saturating_add(i64::from(digit - b'0'))
1592    });
1593    Some(if negative { -magnitude } else { magnitude })
1594}
1595
1596/// How many elements deep the deepest node of `tree` is. A loop over the tree's edges,
1597/// never a recursion.
1598fn html_tree_depth(tree: &ego_tree::Tree<Node>) -> usize {
1599    use ego_tree::iter::Edge;
1600    let mut depth = 0usize;
1601    let mut deepest = 0usize;
1602    for edge in tree.root().traverse() {
1603        match edge {
1604            Edge::Open(node) if node.value().is_element() => {
1605                depth += 1;
1606                deepest = deepest.max(depth);
1607            }
1608            Edge::Close(node) if node.value().is_element() => depth = depth.saturating_sub(1),
1609            _ => {}
1610        }
1611    }
1612    deepest
1613}
1614
1615/// Read `html` into blocks and tables.
1616///
1617/// The reader recurses once per element level it follows, up to `MAX_HTML_READ_DEPTH`,
1618/// and a stack overflow aborts the process: a debug build ran out of a spawned thread's
1619/// stack on a paste nested some 200 elements deep, and a host reading on a smaller thread
1620/// sooner. A tree nested deeper than a caller's stack is sure to hold is read on a thread
1621/// of its own, with a stack sized for its depth; when no thread can be started, it is read
1622/// on the caller's stack to `HTML_DEPTH_ON_THE_CALLERS_STACK` levels, each subtree
1623/// deeper than that read as its text. The quotations it holds are kept within
1624/// [`MAX_QUOTE_LEVELS`](crate::parser_tools::quote_depth::MAX_QUOTE_LEVELS) (see
1625/// `quote_depth`).
1626pub fn parse_html_elements(html: &str) -> Vec<ParsedElement> {
1627    use scraper::Html;
1628
1629    let read = |depth_limit: usize, html: &str| {
1630        let mut fragment = Html::parse_fragment(html);
1631        flatten_deep_html_lists(&mut fragment.tree);
1632        let mut elements = read_html_tree(&fragment, depth_limit);
1633        crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1634        elements
1635    };
1636    let mut fragment = Html::parse_fragment(html);
1637    flatten_deep_html_lists(&mut fragment.tree);
1638    let depth = html_tree_depth(&fragment.tree);
1639    if depth <= HTML_DEPTH_ON_THE_CALLERS_STACK {
1640        let mut elements = read_html_tree(&fragment, MAX_HTML_READ_DEPTH);
1641        crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1642        return elements;
1643    }
1644    drop(fragment);
1645    let stack = HTML_READER_STACK + depth.min(MAX_HTML_READ_DEPTH + 1) * HTML_STACK_PER_LEVEL;
1646    let owned = html.to_owned();
1647    let on_its_own_thread = std::thread::Builder::new()
1648        .stack_size(stack)
1649        .spawn(move || read(MAX_HTML_READ_DEPTH, &owned))
1650        .ok()
1651        .and_then(|reader| reader.join().ok());
1652    on_its_own_thread.unwrap_or_else(|| read(HTML_DEPTH_ON_THE_CALLERS_STACK, html))
1653}
1654
1655/// Read `fragment`, following its elements `depth_limit` deep at most: a subtree nested
1656/// deeper is read as its text (see `subtree_text`). The walk recurses once per element
1657/// level it follows.
1658fn read_html_tree(fragment: &scraper::Html, depth_limit: usize) -> Vec<ParsedElement> {
1659    let mut elements: Vec<ParsedElement> = Vec::new();
1660
1661    // Walk the DOM tree starting from the root
1662    let root = fragment.root_element();
1663
1664    #[derive(Clone, Default)]
1665    struct FmtState {
1666        bold: bool,
1667        italic: bool,
1668        underline: bool,
1669        strikeout: bool,
1670        code: bool,
1671        superscript: bool,
1672        subscript: bool,
1673        link_href: Option<String>,
1674    }
1675
1676    /// The list an element stands in: its style, and the number its first item wears when
1677    /// it is not 1 (`<ol start="3">`).
1678    type HtmlList = (Option<ListStyle>, Option<i64>);
1679
1680    /// Elements whose text content is machinery, not prose.
1681    ///
1682    /// `<style>` and `<script>` hold raw text the HTML spec never renders, and
1683    /// `<head>` and its metadata children carry none a reader is meant to see.
1684    /// Every walker below recurses into unknown tags on the assumption that
1685    /// they are inline wrappers, which turns a stylesheet into paragraphs —
1686    /// and Word, Google Docs and Chrome all put a `<style>` block in the
1687    /// `text/html` flavour they publish to the clipboard, so this is the
1688    /// ordinary paste, not an exotic one.
1689    fn is_metadata_tag(tag: &str) -> bool {
1690        matches!(
1691            tag,
1692            "head"
1693                | "style"
1694                | "script"
1695                | "title"
1696                | "meta"
1697                | "link"
1698                | "base"
1699                | "noscript"
1700                | "template"
1701        )
1702    }
1703
1704    /// The text of everything under `node`, for a subtree nested deeper than
1705    /// the reader follows (see [`MAX_HTML_READ_DEPTH`]): read without recursing,
1706    /// with a space where a block ends so that words do not run together, and
1707    /// without what [`is_metadata_tag`] names.
1708    ///
1709    /// Past that depth the reader used to stop and keep nothing: text nested
1710    /// deeply enough in lists, quotations or plain `<div>`s was dropped from a
1711    /// paste without a trace.
1712    fn subtree_text(node: ego_tree::NodeRef<Node>) -> String {
1713        use ego_tree::iter::Edge;
1714        let mut text = String::new();
1715        // How many elements deep the walk is inside a metadata element.
1716        let mut hidden = 0usize;
1717        for edge in node.traverse() {
1718            match edge {
1719                Edge::Open(open) => match open.value() {
1720                    Node::Element(el) => {
1721                        if hidden > 0 || is_metadata_tag(el.name()) {
1722                            hidden += 1;
1723                        } else if !text.is_empty() && !text.ends_with(char::is_whitespace) {
1724                            text.push(' ');
1725                        }
1726                    }
1727                    Node::Text(t) if hidden == 0 => text.push_str(t),
1728                    _ => {}
1729                },
1730                Edge::Close(close) => {
1731                    if hidden > 0 && close.value().is_element() {
1732                        hidden -= 1;
1733                    }
1734                }
1735            }
1736        }
1737        text
1738    }
1739
1740    /// A span of `text` in `state`'s formatting.
1741    fn text_span(text: String, state: &FmtState) -> ParsedSpan {
1742        ParsedSpan {
1743            text,
1744            bold: state.bold,
1745            italic: state.italic,
1746            underline: state.underline,
1747            strikeout: state.strikeout,
1748            code: state.code,
1749            superscript: state.superscript,
1750            subscript: state.subscript,
1751            link_href: state.link_href.clone(),
1752            image: None,
1753            footnote_ref: None,
1754        }
1755    }
1756
1757    /// Collect inline spans from a `<td>` or `<th>` cell element.
1758    fn collect_cell_spans(
1759        node: ego_tree::NodeRef<Node>,
1760        state: &FmtState,
1761        spans: &mut Vec<ParsedSpan>,
1762        depth: usize,
1763        limit: usize,
1764    ) {
1765        if depth > limit {
1766            let text = subtree_text(node);
1767            if !text.is_empty() {
1768                spans.push(text_span(text, state));
1769            }
1770            return;
1771        }
1772        for child in node.children() {
1773            match child.value() {
1774                Node::Text(text) => {
1775                    let t = text.text.to_string();
1776                    if !t.is_empty() {
1777                        spans.push(ParsedSpan {
1778                            text: t,
1779                            bold: state.bold,
1780                            italic: state.italic,
1781                            underline: state.underline,
1782                            strikeout: state.strikeout,
1783                            code: state.code,
1784                            superscript: state.superscript,
1785                            subscript: state.subscript,
1786                            link_href: state.link_href.clone(),
1787                            image: None,
1788                            footnote_ref: None,
1789                        });
1790                    }
1791                }
1792                Node::Element(el) => {
1793                    let tag = el.name();
1794                    if is_metadata_tag(tag) {
1795                        continue;
1796                    }
1797                    let mut new_state = state.clone();
1798                    match tag {
1799                        // Checked by attribute rather than by tag name: the
1800                        // producer picks the element (a `<sup>` renders sensibly
1801                        // in a browser), and only the attribute is a contract.
1802                        _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
1803                            if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
1804                            {
1805                                spans.push(span);
1806                            }
1807                            continue;
1808                        }
1809                        "b" | "strong" => new_state.bold = true,
1810                        "i" | "em" => new_state.italic = true,
1811                        "u" | "ins" => new_state.underline = true,
1812                        "s" | "del" | "strike" => new_state.strikeout = true,
1813                        "code" => new_state.code = true,
1814                        "sup" => new_state.superscript = true,
1815                        "sub" => new_state.subscript = true,
1816                        "a" => {
1817                            if let Some(href) = el.attr("href") {
1818                                new_state.link_href = Some(href.to_string());
1819                            }
1820                        }
1821                        "img" => {
1822                            if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
1823                                spans.push(span);
1824                            }
1825                            continue;
1826                        }
1827                        // A cell is read as one paragraph, so a line break, or a
1828                        // paragraph or list item of its own, is a space between
1829                        // words. Read as nothing, a cell of two paragraphs, which
1830                        // the HTML writer puts out as `One<br/>Two`, came back as
1831                        // the one word `OneTwo`.
1832                        "br" => {
1833                            spans.push(text_span(" ".to_string(), state));
1834                            continue;
1835                        }
1836                        "p" | "div" | "li" | "ul" | "ol" | "blockquote" | "pre" | "h1" | "h2"
1837                        | "h3" | "h4" | "h5" | "h6" => {
1838                            spans.push(text_span(" ".to_string(), state));
1839                        }
1840                        _ => {}
1841                    }
1842                    collect_cell_spans(child, &new_state, spans, depth + 1, limit);
1843                }
1844                _ => {}
1845            }
1846        }
1847    }
1848
1849    /// The widest a `colspan` makes a row: HTML's own limit on the attribute. A row of
1850    /// ordinary cells may be wider; only the empty cells a span adds stop here.
1851    const MAX_HTML_COLSPAN: usize = 1000;
1852
1853    /// Parse a `<table>` element into a ParsedTable, with the number of `<td>` and `<th>`
1854    /// cells it has (see `push_table`). `depth` is how deep the walk stands at the table,
1855    /// and `limit` how deep it goes.
1856    fn parse_table_element(
1857        table_node: ego_tree::NodeRef<Node>,
1858        depth: usize,
1859        limit: usize,
1860    ) -> (ParsedTable, usize) {
1861        let mut rows: Vec<Vec<ParsedTableCell>> = Vec::new();
1862        let mut header_rows: usize = 0;
1863        let mut own_cells: usize = 0;
1864
1865        /// `walk` is how deep the walk stands at `node` and how deep it goes.
1866        fn collect_rows(
1867            node: ego_tree::NodeRef<Node>,
1868            rows: &mut Vec<Vec<ParsedTableCell>>,
1869            header_rows: &mut usize,
1870            own_cells: &mut usize,
1871            in_thead: bool,
1872            walk: (usize, usize),
1873        ) {
1874            let (depth, limit) = walk;
1875            for child in node.children() {
1876                if let Node::Element(el) = child.value() {
1877                    match el.name() {
1878                        "thead" => collect_rows(
1879                            child,
1880                            rows,
1881                            header_rows,
1882                            own_cells,
1883                            true,
1884                            (depth + 1, limit),
1885                        ),
1886                        "tbody" | "tfoot" => collect_rows(
1887                            child,
1888                            rows,
1889                            header_rows,
1890                            own_cells,
1891                            false,
1892                            (depth + 1, limit),
1893                        ),
1894                        "tr" => {
1895                            let mut cells: Vec<ParsedTableCell> = Vec::new();
1896                            // The cells of the row up to its last `<td>` or `<th>`: the
1897                            // empty cells a span adds after it hold no cell's place.
1898                            let mut own_end = 0;
1899                            for td in child.children() {
1900                                if let Node::Element(td_el) = td.value()
1901                                    && matches!(td_el.name(), "td" | "th")
1902                                {
1903                                    let mut spans = Vec::new();
1904                                    let state = FmtState::default();
1905                                    collect_cell_spans(td, &state, &mut spans, depth + 2, limit);
1906                                    collapse_inline_whitespace(&mut spans);
1907                                    if spans.is_empty() {
1908                                        spans.push(ParsedSpan::default());
1909                                    }
1910                                    cells.push(ParsedTableCell { spans });
1911                                    *own_cells += 1;
1912                                    own_end = cells.len();
1913                                    // A cell spanning columns covers the cells after it
1914                                    // in its row: empty ones hold its place, so the cells
1915                                    // that follow stay in their own columns and the
1916                                    // table is as wide as the columns it spans.
1917                                    let span = td_el
1918                                        .attr("colspan")
1919                                        .and_then(|value| value.trim().parse::<usize>().ok())
1920                                        .unwrap_or(1)
1921                                        .clamp(1, MAX_HTML_COLSPAN);
1922                                    for _ in 1..span {
1923                                        if cells.len() >= MAX_HTML_COLSPAN {
1924                                            break;
1925                                        }
1926                                        cells.push(ParsedTableCell {
1927                                            spans: vec![ParsedSpan::default()],
1928                                        });
1929                                    }
1930                                }
1931                            }
1932                            // A span closing a row widens the table only as far as another
1933                            // row's cells reach: `<th colspan="1000">` over rows of one
1934                            // cell is a table of one column, not of a thousand.
1935                            cells.truncate(own_end);
1936                            if !cells.is_empty() {
1937                                rows.push(cells);
1938                                if in_thead {
1939                                    *header_rows += 1;
1940                                }
1941                            }
1942                        }
1943                        _ => {}
1944                    }
1945                }
1946            }
1947        }
1948
1949        collect_rows(
1950            table_node,
1951            &mut rows,
1952            &mut header_rows,
1953            &mut own_cells,
1954            false,
1955            (depth, limit),
1956        );
1957
1958        // Tables without explicit <thead> but with <th> cells: treat first row as header
1959        if header_rows == 0 && !rows.is_empty() {
1960            header_rows = 1;
1961        }
1962
1963        (
1964            ParsedTable {
1965                header_rows,
1966                rows,
1967                // The caller (`walk_node`) sets the real depth: this helper has
1968                // no visibility into the surrounding blockquote nesting.
1969                blockquote_depth: 0,
1970                blockquote_opens: 0,
1971            },
1972            own_cells,
1973        )
1974    }
1975
1976    #[allow(clippy::too_many_arguments)]
1977    fn walk_node(
1978        node: ego_tree::NodeRef<Node>,
1979        state: &FmtState,
1980        elements: &mut Vec<ParsedElement>,
1981        current_list: &HtmlList,
1982        blockquote_depth: u32,
1983        list_depth: u32,
1984        depth: usize,
1985        limit: usize,
1986    ) {
1987        if depth > limit {
1988            let text = subtree_text(node);
1989            if !text.trim().is_empty() {
1990                let mut spans = vec![text_span(text, state)];
1991                collapse_inline_whitespace(&mut spans);
1992                elements.push(ParsedElement::Block(ParsedBlock {
1993                    spans,
1994                    blockquote_depth,
1995                    blockquote_opens: 0,
1996                    ..ParsedBlock::default()
1997                }));
1998            }
1999            return;
2000        }
2001        match node.value() {
2002            Node::Element(el) => {
2003                let tag = el.name();
2004                if is_metadata_tag(tag) {
2005                    return;
2006                }
2007                let mut new_state = state.clone();
2008                let mut new_list = current_list.clone();
2009                let mut bq_depth = blockquote_depth;
2010                let mut new_list_depth = list_depth;
2011
2012                // Determine if this is a block-level element
2013                let is_block_tag = matches!(
2014                    tag,
2015                    "p" | "div"
2016                        | "h1"
2017                        | "h2"
2018                        | "h3"
2019                        | "h4"
2020                        | "h5"
2021                        | "h6"
2022                        | "li"
2023                        | "pre"
2024                        | "br"
2025                        | "blockquote"
2026                        | "body"
2027                        | "html"
2028                );
2029
2030                // Update formatting state
2031                match tag {
2032                    "b" | "strong" => new_state.bold = true,
2033                    "i" | "em" => new_state.italic = true,
2034                    "u" | "ins" => new_state.underline = true,
2035                    "s" | "del" | "strike" => new_state.strikeout = true,
2036                    "code" => new_state.code = true,
2037                    "sup" => new_state.superscript = true,
2038                    "sub" => new_state.subscript = true,
2039                    "a" => {
2040                        if let Some(href) = el.attr("href") {
2041                            new_state.link_href = Some(href.to_string());
2042                        }
2043                    }
2044                    "ul" => {
2045                        new_list = (Some(ListStyle::Disc), None);
2046                        new_list_depth = list_depth + 1;
2047                    }
2048                    "ol" => {
2049                        // A list numbered from below 0 is kept from 1: no saved syntax
2050                        // but HTML writes one. A start past what every format holds is
2051                        // read as the largest it holds (see `list_start`).
2052                        let start = el
2053                            .attr("start")
2054                            .and_then(html_integer)
2055                            .and_then(crate::parser_tools::list_start::list_start);
2056                        new_list = (Some(ListStyle::Decimal), start);
2057                        new_list_depth = list_depth + 1;
2058                    }
2059                    "blockquote" => {
2060                        bq_depth += 1;
2061                    }
2062                    _ => {}
2063                }
2064
2065                // Determine heading level
2066                let heading_level = match tag {
2067                    "h1" => Some(1),
2068                    "h2" => Some(2),
2069                    "h3" => Some(3),
2070                    "h4" => Some(4),
2071                    "h5" => Some(5),
2072                    "h6" => Some(6),
2073                    _ => None,
2074                };
2075
2076                let is_code_block = tag == "pre";
2077
2078                // Extract code language from <pre><code class="language-xxx">
2079                let code_language = if is_code_block {
2080                    node.children().find_map(|child| {
2081                        if let Node::Element(cel) = child.value()
2082                            && cel.name() == "code"
2083                            && let Some(cls) = cel.attr("class")
2084                        {
2085                            return cls
2086                                .split_whitespace()
2087                                .find_map(|c| c.strip_prefix("language-"))
2088                                .map(|l| l.to_string());
2089                        }
2090                        None
2091                    })
2092                } else {
2093                    None
2094                };
2095
2096                // Extract CSS styles from block-level elements
2097                let css = if is_block_tag {
2098                    el.attr("style").map(parse_block_styles).unwrap_or_default()
2099                } else {
2100                    BlockStyles::default()
2101                };
2102
2103                if tag == "table" {
2104                    // Parse table structure into a ParsedTable
2105                    let (mut parsed_table, own_cells) = parse_table_element(node, depth, limit);
2106                    if !parsed_table.rows.is_empty() {
2107                        parsed_table.blockquote_depth = bq_depth;
2108                        push_table(elements, parsed_table, own_cells);
2109                    }
2110                    return;
2111                }
2112
2113                if tag == "br" {
2114                    // <br> creates a new block
2115                    elements.push(ParsedElement::Block(ParsedBlock {
2116                        spans: vec![ParsedSpan {
2117                            text: String::new(),
2118                            ..Default::default()
2119                        }],
2120                        heading_level: None,
2121                        list_style: None,
2122                        list_indent: 0,
2123                        list_prefix: String::new(),
2124                        list_suffix: String::new(),
2125                        list_start: None,
2126                        marker: None,
2127                        is_code_block: false,
2128                        code_language: None,
2129                        blockquote_depth: bq_depth,
2130                        blockquote_opens: 0,
2131                        line_height: None,
2132                        non_breakable_lines: None,
2133                        page_break_before: None,
2134                        front_matter: None,
2135                        direction: None,
2136                        background_color: None,
2137                        alignment: None,
2138                        top_margin: None,
2139                        text_indent: None,
2140                        semantic_role: None,
2141                    }));
2142                    return;
2143                }
2144
2145                if tag == "blockquote" {
2146                    let first = elements.len();
2147                    // A quotation holding inline content alone is one paragraph of it, as
2148                    // a `<p>` is, its `<br>`s line breaks: verse is pasted from the web as
2149                    // `<blockquote>a line<br>the next</blockquote>`. Walked child by child,
2150                    // each line was a paragraph of its own and each `<br>` an empty one.
2151                    let inline_only = node.children().all(|child| match child.value() {
2152                        Node::Element(el) => !matches!(
2153                            el.name(),
2154                            "p" | "div"
2155                                | "h1"
2156                                | "h2"
2157                                | "h3"
2158                                | "h4"
2159                                | "h5"
2160                                | "h6"
2161                                | "li"
2162                                | "pre"
2163                                | "blockquote"
2164                                | "ul"
2165                                | "ol"
2166                                | "table"
2167                        ),
2168                        _ => true,
2169                    });
2170                    if inline_only {
2171                        let mut spans: Vec<ParsedSpan> = Vec::new();
2172                        let mut nested_elements: Vec<ParsedElement> = Vec::new();
2173                        collect_inline_spans(
2174                            node,
2175                            &new_state,
2176                            &mut spans,
2177                            &new_list,
2178                            &mut nested_elements,
2179                            bq_depth,
2180                            new_list_depth,
2181                            depth + 1,
2182                            limit,
2183                        );
2184                        collapse_inline_whitespace(&mut spans);
2185                        if spans_carry_content(&spans) {
2186                            elements.push(ParsedElement::Block(ParsedBlock {
2187                                spans,
2188                                blockquote_depth: bq_depth,
2189                                blockquote_opens: 0,
2190                                ..ParsedBlock::default()
2191                            }));
2192                        }
2193                        elements.append(&mut nested_elements);
2194                    } else {
2195                        // A container: recurse into children with increased depth.
2196                        for child in node.children() {
2197                            walk_node(
2198                                child,
2199                                &new_state,
2200                                elements,
2201                                &new_list,
2202                                bq_depth,
2203                                new_list_depth,
2204                                depth + 1,
2205                                limit,
2206                            );
2207                        }
2208                    }
2209                    mark_quotation_opened(elements, first);
2210                } else if is_block_tag && tag != "br" {
2211                    // Start collecting spans for a new block.
2212                    // Use a temporary buffer so that nested block-level
2213                    // elements (e.g. sub-lists inside <li>) are collected
2214                    // separately and appended *after* the parent block.
2215                    let mut spans: Vec<ParsedSpan> = Vec::new();
2216                    let mut nested_elements: Vec<ParsedElement> = Vec::new();
2217                    collect_inline_spans(
2218                        node,
2219                        &new_state,
2220                        &mut spans,
2221                        &new_list,
2222                        &mut nested_elements,
2223                        bq_depth,
2224                        new_list_depth,
2225                        depth + 1,
2226                        limit,
2227                    );
2228
2229                    let (list_style_for_block, list_start_for_block) = if tag == "li" {
2230                        new_list.clone()
2231                    } else {
2232                        (None, None)
2233                    };
2234
2235                    let list_indent_for_block = if tag == "li" {
2236                        new_list_depth.saturating_sub(1)
2237                    } else {
2238                        0
2239                    };
2240
2241                    if !(is_code_block || css.preserve_whitespace == Some(true)) {
2242                        collapse_inline_whitespace(&mut spans);
2243                    }
2244
2245                    // Whitespace between block-level children is layout, not
2246                    // content. `<body>` holds a newline after every `</p>` an
2247                    // exporter writes, and keeping those as the container's own
2248                    // inline run is what closed a pasted book with a paragraph
2249                    // of four thousand blank lines. A block that produced no
2250                    // children keeps its empty run instead: `<p> </p>` is an
2251                    // empty paragraph the writer meant to be there.
2252                    let own_run_is_content =
2253                        spans_carry_content(&spans) || nested_elements.is_empty();
2254
2255                    if (!spans.is_empty() && own_run_is_content) || heading_level.is_some() {
2256                        // A code block is verbatim text, which no save can write an image or
2257                        // a note's reference into: a `<pre>` holding one is read as prose,
2258                        // which keeps it, one paragraph for each of its lines (see below).
2259                        let is_code_block = is_code_block
2260                            && !spans
2261                                .iter()
2262                                .any(|span| span.image.is_some() || span.footnote_ref.is_some());
2263                        let block = ParsedBlock {
2264                            spans,
2265                            heading_level,
2266                            list_style: list_style_for_block,
2267                            list_indent: list_indent_for_block,
2268                            list_prefix: String::new(),
2269                            list_suffix: String::new(),
2270                            list_start: list_start_for_block,
2271                            marker: None,
2272                            is_code_block,
2273                            code_language: code_language.filter(|_| is_code_block),
2274                            blockquote_depth: bq_depth,
2275                            blockquote_opens: 0,
2276                            line_height: css.line_height,
2277                            non_breakable_lines: css.non_breakable_lines,
2278                            page_break_before: css.page_break_before,
2279                            front_matter: None,
2280                            direction: css.direction,
2281                            background_color: css.background_color,
2282                            alignment: None,
2283                            top_margin: None,
2284                            text_indent: None,
2285                            semantic_role: None,
2286                        };
2287                        // Kept whitespace keeps its line breaks, and a line break in
2288                        // prose is a new block to the model. Only a code block holds
2289                        // its lines in one block.
2290                        if is_code_block {
2291                            elements.push(ParsedElement::Block(block));
2292                        } else {
2293                            elements.extend(
2294                                split_block_at_line_breaks(block)
2295                                    .into_iter()
2296                                    .map(ParsedElement::Block),
2297                            );
2298                        }
2299                    }
2300                    // Append nested block elements after the parent block
2301                    elements.append(&mut nested_elements);
2302                } else if matches!(tag, "ul" | "ol" | "thead" | "tbody" | "tr") {
2303                    // Container elements: recurse into children
2304                    for child in node.children() {
2305                        walk_node(
2306                            child,
2307                            &new_state,
2308                            elements,
2309                            &new_list,
2310                            bq_depth,
2311                            new_list_depth,
2312                            depth + 1,
2313                            limit,
2314                        );
2315                    }
2316                } else {
2317                    // Inline element or unknown: recurse
2318                    for child in node.children() {
2319                        walk_node(
2320                            child,
2321                            &new_state,
2322                            elements,
2323                            current_list,
2324                            bq_depth,
2325                            list_depth,
2326                            depth + 1,
2327                            limit,
2328                        );
2329                    }
2330                }
2331            }
2332            Node::Text(text) => {
2333                let t = text.text.to_string();
2334                let trimmed = t.trim();
2335                if !trimmed.is_empty() {
2336                    // Bare text not in a block — create a paragraph
2337                    elements.push(ParsedElement::Block(ParsedBlock {
2338                        spans: vec![ParsedSpan {
2339                            text: trimmed.to_string(),
2340                            bold: state.bold,
2341                            italic: state.italic,
2342                            underline: state.underline,
2343                            strikeout: state.strikeout,
2344                            code: state.code,
2345                            superscript: state.superscript,
2346                            subscript: state.subscript,
2347                            link_href: state.link_href.clone(),
2348                            image: None,
2349                            footnote_ref: None,
2350                        }],
2351                        heading_level: None,
2352                        list_style: None,
2353                        list_indent: 0,
2354                        list_prefix: String::new(),
2355                        list_suffix: String::new(),
2356                        list_start: None,
2357                        marker: None,
2358                        is_code_block: false,
2359                        code_language: None,
2360                        blockquote_depth,
2361                        blockquote_opens: 0,
2362                        line_height: None,
2363                        non_breakable_lines: None,
2364                        page_break_before: None,
2365                        front_matter: None,
2366                        direction: None,
2367                        background_color: None,
2368                        alignment: None,
2369                        top_margin: None,
2370                        text_indent: None,
2371                        semantic_role: None,
2372                    }));
2373                }
2374            }
2375            _ => {
2376                // Document, Comment, etc. — recurse children
2377                for child in node.children() {
2378                    walk_node(
2379                        child,
2380                        state,
2381                        elements,
2382                        current_list,
2383                        blockquote_depth,
2384                        list_depth,
2385                        depth + 1,
2386                        limit,
2387                    );
2388                }
2389            }
2390        }
2391    }
2392
2393    /// Collect inline spans from a block-level element's children.
2394    /// If a nested block-level element is encountered, it is flushed as a
2395    /// separate block.
2396    #[allow(clippy::too_many_arguments)]
2397    fn collect_inline_spans(
2398        node: ego_tree::NodeRef<Node>,
2399        state: &FmtState,
2400        spans: &mut Vec<ParsedSpan>,
2401        current_list: &HtmlList,
2402        elements: &mut Vec<ParsedElement>,
2403        blockquote_depth: u32,
2404        list_depth: u32,
2405        depth: usize,
2406        limit: usize,
2407    ) {
2408        if depth > limit {
2409            let text = subtree_text(node);
2410            if !text.is_empty() {
2411                spans.push(text_span(text, state));
2412            }
2413            return;
2414        }
2415        for child in node.children() {
2416            match child.value() {
2417                Node::Text(text) => {
2418                    let t = text.text.to_string();
2419                    if !t.is_empty() {
2420                        spans.push(ParsedSpan {
2421                            text: t,
2422                            bold: state.bold,
2423                            italic: state.italic,
2424                            underline: state.underline,
2425                            strikeout: state.strikeout,
2426                            code: state.code,
2427                            superscript: state.superscript,
2428                            subscript: state.subscript,
2429                            link_href: state.link_href.clone(),
2430                            image: None,
2431                            footnote_ref: None,
2432                        });
2433                    }
2434                }
2435                Node::Element(el) => {
2436                    let tag = el.name();
2437                    if is_metadata_tag(tag) {
2438                        continue;
2439                    }
2440                    let mut new_state = state.clone();
2441
2442                    match tag {
2443                        // Checked by attribute rather than by tag name: the
2444                        // producer picks the element (a `<sup>` renders sensibly
2445                        // in a browser), and only the attribute is a contract.
2446                        _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
2447                            if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
2448                            {
2449                                spans.push(span);
2450                            }
2451                            continue;
2452                        }
2453                        "b" | "strong" => new_state.bold = true,
2454                        "i" | "em" => new_state.italic = true,
2455                        "u" | "ins" => new_state.underline = true,
2456                        "s" | "del" | "strike" => new_state.strikeout = true,
2457                        "code" => new_state.code = true,
2458                        "sup" => new_state.superscript = true,
2459                        "sub" => new_state.subscript = true,
2460                        "a" => {
2461                            if let Some(href) = el.attr("href") {
2462                                new_state.link_href = Some(href.to_string());
2463                            }
2464                        }
2465                        "img" => {
2466                            if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
2467                                spans.push(span);
2468                            }
2469                            continue;
2470                        }
2471                        _ => {}
2472                    }
2473
2474                    // Check for nested block elements
2475                    let nested_block = matches!(
2476                        tag,
2477                        "p" | "div"
2478                            | "h1"
2479                            | "h2"
2480                            | "h3"
2481                            | "h4"
2482                            | "h5"
2483                            | "h6"
2484                            | "li"
2485                            | "pre"
2486                            | "blockquote"
2487                            | "ul"
2488                            | "ol"
2489                    );
2490
2491                    if tag == "br" {
2492                        // A line break inside the block, in the formatting around it.
2493                        // `collapse_inline_whitespace` then drops the spaces either side,
2494                        // as a browser lays them out.
2495                        spans.push(text_span(LINE_SEPARATOR_STR.to_string(), state));
2496                    } else if nested_block || tag == "table" {
2497                        // Flush as separate element
2498                        walk_node(
2499                            child,
2500                            &new_state,
2501                            elements,
2502                            current_list,
2503                            blockquote_depth,
2504                            list_depth,
2505                            depth + 1,
2506                            limit,
2507                        );
2508                    } else {
2509                        // Inline element: recurse
2510                        collect_inline_spans(
2511                            child,
2512                            &new_state,
2513                            spans,
2514                            current_list,
2515                            elements,
2516                            blockquote_depth,
2517                            list_depth,
2518                            depth + 1,
2519                            limit,
2520                        );
2521                    }
2522                }
2523                _ => {}
2524            }
2525        }
2526    }
2527
2528    let initial_state = FmtState::default();
2529    // Treat the root element as a block-level container so that
2530    // top-level inline elements (e.g. `<b>Bold</b> <em>Italic</em>`)
2531    // are grouped into a single block instead of becoming separate blocks.
2532    let mut root_spans: Vec<ParsedSpan> = Vec::new();
2533    collect_inline_spans(
2534        *root,
2535        &initial_state,
2536        &mut root_spans,
2537        &(None, None),
2538        &mut elements,
2539        0,
2540        0,
2541        0,
2542        depth_limit,
2543    );
2544    collapse_inline_whitespace(&mut root_spans);
2545    // Only when something survived: the root's direct text children are the
2546    // newlines between top-level blocks in every document an exporter writes.
2547    if spans_carry_content(&root_spans) {
2548        elements.push(ParsedElement::Block(ParsedBlock {
2549            spans: root_spans,
2550            heading_level: None,
2551            list_style: None,
2552            list_indent: 0,
2553            list_prefix: String::new(),
2554            list_suffix: String::new(),
2555            list_start: None,
2556            marker: None,
2557            is_code_block: false,
2558            code_language: None,
2559            blockquote_depth: 0,
2560            blockquote_opens: 0,
2561            line_height: None,
2562            non_breakable_lines: None,
2563            page_break_before: None,
2564            front_matter: None,
2565            direction: None,
2566            background_color: None,
2567            alignment: None,
2568            top_margin: None,
2569            text_indent: None,
2570            semantic_role: None,
2571        }));
2572    }
2573
2574    // If no elements were parsed, create a single empty paragraph
2575    if elements.is_empty() {
2576        elements.push(ParsedElement::Block(ParsedBlock {
2577            spans: vec![ParsedSpan {
2578                text: String::new(),
2579                ..Default::default()
2580            }],
2581            heading_level: None,
2582            list_style: None,
2583            list_indent: 0,
2584            list_prefix: String::new(),
2585            list_suffix: String::new(),
2586            list_start: None,
2587            marker: None,
2588            is_code_block: false,
2589            code_language: None,
2590            blockquote_depth: 0,
2591            blockquote_opens: 0,
2592            line_height: None,
2593            non_breakable_lines: None,
2594            page_break_before: None,
2595            front_matter: None,
2596            direction: None,
2597            background_color: None,
2598            alignment: None,
2599            top_margin: None,
2600            text_indent: None,
2601            semantic_role: None,
2602        }));
2603    }
2604
2605    elements
2606}
2607
2608/// Convert a `ParsedSpan` (parser output) into the `CharacterFormat` used by
2609/// `FormatRun`. `is_code_block` forces `monospace` as the font family for
2610/// every span inside a code block.
2611pub fn character_format_from_span(
2612    span: &ParsedSpan,
2613    is_code_block: bool,
2614) -> crate::format_runs::CharacterFormat {
2615    use crate::entities::CharVerticalAlignment;
2616    crate::format_runs::CharacterFormat {
2617        font_bold: if span.bold { Some(true) } else { None },
2618        font_italic: if span.italic { Some(true) } else { None },
2619        font_underline: if span.underline { Some(true) } else { None },
2620        font_strikeout: if span.strikeout { Some(true) } else { None },
2621        font_family: if span.code || is_code_block {
2622            Some("monospace".to_string())
2623        } else {
2624            None
2625        },
2626        anchor_href: span.link_href.clone(),
2627        is_anchor: if span.link_href.is_some() {
2628            Some(true)
2629        } else {
2630            None
2631        },
2632        vertical_alignment: if span.superscript {
2633            Some(CharVerticalAlignment::SuperScript)
2634        } else if span.subscript {
2635            Some(CharVerticalAlignment::SubScript)
2636        } else {
2637            None
2638        },
2639        ..Default::default()
2640    }
2641}
2642
2643/// Translate a slice of parsed spans into `(plain_text, format_runs)`.
2644///
2645/// One non-default span yields one `FormatRun`; spans with empty
2646/// `CharacterFormat` (no decoration, no link, no code) emit no run, since an
2647/// absent run means "inherit default formatting" in the new model. Adjacent
2648/// runs with identical formats are coalesced via `coalesce_in_place` so the
2649/// resulting vector satisfies `debug_assert_well_formed`.
2650///
2651/// Returns the concatenated `plain_text` of all spans and a sorted,
2652/// non-overlapping, coalesced `Vec<FormatRun>`. Both safe to feed straight
2653/// into the store under the dual-write bridge.
2654pub fn format_runs_from_spans(spans: &[ParsedSpan], is_code_block: bool) -> ParsedInline {
2655    use crate::format_runs::{
2656        CharacterFormat, FootnoteRefAnchor, FormatRun, ImageAnchor, coalesce_in_place,
2657    };
2658
2659    let mut plain_text = String::new();
2660    let mut runs: Vec<FormatRun> = Vec::new();
2661    let mut images: Vec<ImageAnchor> = Vec::new();
2662    let mut footnote_refs: Vec<FootnoteRefAnchor> = Vec::new();
2663    let default = CharacterFormat::default();
2664
2665    for span in spans {
2666        let byte_start = plain_text.len() as u32;
2667
2668        if let Some(label) = &span.footnote_ref {
2669            // A reference occupies one U+FFFC, exactly as an image does, so
2670            // every downstream offset treats the two alike.
2671            plain_text.push('\u{FFFC}');
2672            // Raised, always — whatever the surrounding run is doing.
2673            //
2674            // A footnote marker is superscript by definition, in every
2675            // typographic tradition and in every reader that renders djot. The
2676            // ambient `superscript` flag the span carries is the *prose's*, and
2677            // prose is not superscript, so taking it verbatim sets a note's
2678            // number on the baseline in the middle of a sentence — which reads
2679            // as a stray digit the writer typed rather than as a reference.
2680            //
2681            // It is set here, on the anchor, rather than at render time so that
2682            // every consumer agrees: the editor raises it, the exporters that
2683            // carry character formatting carry it, and the djot writer knows to
2684            // emit `[^label]` *without* wrapping it in `^…^` (see
2685            // `a_reference_is_not_wrapped_in_superscript_markers`).
2686            let mut format = character_format_from_span(span, is_code_block);
2687            format.vertical_alignment = Some(crate::entities::CharVerticalAlignment::SuperScript);
2688            footnote_refs.push(FootnoteRefAnchor {
2689                byte_offset: byte_start,
2690                label: label.clone(),
2691                format,
2692            });
2693            continue;
2694        }
2695
2696        if let Some(image) = &span.image {
2697            // An image occupies one U+FFFC in the text, exactly as
2698            // `insert_image` mirrors into the rope, so every downstream offset
2699            // calculation treats a parsed image and an inserted one alike.
2700            plain_text.push('\u{FFFC}');
2701            images.push(ImageAnchor {
2702                byte_offset: byte_start,
2703                name: image.src.clone(),
2704                alt: image.alt.clone(),
2705                width: image.width,
2706                height: image.height,
2707                quality: 100,
2708                format: character_format_from_span(span, is_code_block),
2709            });
2710            continue;
2711        }
2712
2713        plain_text.push_str(&span.text);
2714        let byte_end = plain_text.len() as u32;
2715        if byte_start == byte_end {
2716            continue;
2717        }
2718        let format = character_format_from_span(span, is_code_block);
2719        if format == default {
2720            continue;
2721        }
2722        runs.push(FormatRun {
2723            byte_start,
2724            byte_end,
2725            format,
2726        });
2727    }
2728    coalesce_in_place(&mut runs);
2729    ParsedInline {
2730        plain_text,
2731        runs,
2732        images,
2733        footnote_refs,
2734    }
2735}
2736
2737/// The three parallel things a block stores, as recovered from parsed spans.
2738///
2739/// Returned as a struct rather than a tuple because it grew a third member
2740/// (images) after nine call sites already destructured a pair — and every one
2741/// of those sites has to decide what to do with images, so a silent
2742/// tuple-arity change would have been the wrong kind of easy.
2743#[derive(Debug, Clone, Default)]
2744pub struct ParsedInline {
2745    pub plain_text: String,
2746    pub runs: Vec<crate::format_runs::FormatRun>,
2747    pub images: Vec<crate::format_runs::ImageAnchor>,
2748    pub footnote_refs: Vec<crate::format_runs::FootnoteRefAnchor>,
2749}
2750
2751// ─── Djot parsing ────────────────────────────────────────────────────
2752
2753/// Map a jotdown unordered/task bullet marker to a model `ListStyle`.
2754///
2755/// The mapping is a stable bijection (`-`↔Disc, `*`↔Circle, `+`↔Square) so the
2756/// djot exporter can recover the exact bullet character for a lossless
2757/// round-trip.
2758fn djot_bullet_style(b: jotdown::ListBulletType) -> ListStyle {
2759    use jotdown::ListBulletType as B;
2760    match b {
2761        B::Dash => ListStyle::Disc,
2762        B::Star => ListStyle::Circle,
2763        B::Plus => ListStyle::Square,
2764    }
2765}
2766
2767/// Map a jotdown ordered-list numbering scheme to a model `ListStyle`.
2768fn djot_ordered_style(n: jotdown::OrderedListNumbering) -> ListStyle {
2769    use jotdown::OrderedListNumbering as N;
2770    match n {
2771        N::Decimal => ListStyle::Decimal,
2772        N::AlphaLower => ListStyle::LowerAlpha,
2773        N::AlphaUpper => ListStyle::UpperAlpha,
2774        N::RomanLower => ListStyle::LowerRoman,
2775        N::RomanUpper => ListStyle::UpperRoman,
2776    }
2777}
2778
2779/// Map a jotdown ordered-list delimiter to the `(prefix, suffix)` affixes
2780/// stored on the `List` entity (`1.` → `("", ".")`, `1)` → `("", ")")`,
2781/// `(1)` → `("(", ")")`).
2782fn djot_ordered_affixes(style: jotdown::OrderedListStyle) -> (String, String) {
2783    use jotdown::OrderedListStyle as S;
2784    match style {
2785        S::Period => (String::new(), ".".to_string()),
2786        S::Paren => (String::new(), ")".to_string()),
2787        S::ParenParen => ("(".to_string(), ")".to_string()),
2788    }
2789}
2790
2791/// Optional block-level style attributes carried on a djot block through its
2792/// `{key=value}` block attributes. All `None` when the block has no such
2793/// attributes (or they were filtered out by [`DjotImportOptions`]).
2794#[derive(Debug, Clone, Default)]
2795struct DjotBlockStyle {
2796    alignment: Option<Alignment>,
2797    line_height: Option<i64>,
2798    non_breakable_lines: Option<bool>,
2799    page_break_before: Option<bool>,
2800    front_matter: Option<bool>,
2801    direction: Option<TextDirection>,
2802    background_color: Option<String>,
2803    top_margin: Option<i64>,
2804    text_indent: Option<i64>,
2805    semantic_role: Option<SemanticRole>,
2806}
2807
2808impl DjotBlockStyle {
2809    /// Overlay the `Some` fields of `other` onto `self`, leaving `self`'s
2810    /// existing values for any field `other` does not set. Used to combine a
2811    /// heading's enclosing-`Section` attributes with any on the heading itself.
2812    fn merge_from(&mut self, other: DjotBlockStyle) {
2813        if other.alignment.is_some() {
2814            self.alignment = other.alignment;
2815        }
2816        if other.line_height.is_some() {
2817            self.line_height = other.line_height;
2818        }
2819        if other.non_breakable_lines.is_some() {
2820            self.non_breakable_lines = other.non_breakable_lines;
2821        }
2822        if other.page_break_before.is_some() {
2823            self.page_break_before = other.page_break_before;
2824        }
2825        if other.front_matter.is_some() {
2826            self.front_matter = other.front_matter;
2827        }
2828        if other.direction.is_some() {
2829            self.direction = other.direction;
2830        }
2831        if other.background_color.is_some() {
2832            self.background_color = other.background_color;
2833        }
2834        if other.top_margin.is_some() {
2835            self.top_margin = other.top_margin;
2836        }
2837        if other.text_indent.is_some() {
2838            self.text_indent = other.text_indent;
2839        }
2840        if other.semantic_role.is_some() {
2841            self.semantic_role = other.semantic_role.clone();
2842        }
2843    }
2844}
2845
2846/// Read the round-tripped block-style attributes off a djot block's
2847/// [`jotdown::Attributes`], honouring the import [`DjotImportOptions`]. Keys are
2848/// the model field names (`alignment`, `line_height`, `direction`,
2849/// `non_breakable_lines`, `page_break_before`, `front_matter`, `background_color`,
2850/// `top_margin`, `text_indent`, `semantic_role`); unrecognised values are ignored.
2851fn block_attrs_to_style(attrs: &jotdown::Attributes, opts: &DjotImportOptions) -> DjotBlockStyle {
2852    let mut style = DjotBlockStyle::default();
2853
2854    if opts.alignment
2855        && let Some(v) = attrs.get_value("alignment")
2856    {
2857        style.alignment = match v.to_string().as_str() {
2858            "left" => Some(Alignment::Left),
2859            "right" => Some(Alignment::Right),
2860            "center" => Some(Alignment::Center),
2861            "justify" => Some(Alignment::Justify),
2862            _ => None,
2863        };
2864    }
2865    if opts.line_height
2866        && let Some(v) = attrs.get_value("line_height")
2867    {
2868        style.line_height = v.to_string().parse::<i64>().ok();
2869    }
2870    if opts.direction
2871        && let Some(v) = attrs.get_value("direction")
2872    {
2873        style.direction = match v.to_string().as_str() {
2874            "ltr" => Some(TextDirection::LeftToRight),
2875            "rtl" => Some(TextDirection::RightToLeft),
2876            _ => None,
2877        };
2878    }
2879    if opts.non_breakable_lines
2880        && let Some(v) = attrs.get_value("non_breakable_lines")
2881    {
2882        style.non_breakable_lines = match v.to_string().as_str() {
2883            "true" => Some(true),
2884            "false" => Some(false),
2885            _ => None,
2886        };
2887    }
2888    if opts.page_break_before
2889        && let Some(v) = attrs.get_value("page_break_before")
2890    {
2891        style.page_break_before = match v.to_string().as_str() {
2892            "true" => Some(true),
2893            "false" => Some(false),
2894            _ => None,
2895        };
2896    }
2897    if opts.front_matter
2898        && let Some(v) = attrs.get_value("front_matter")
2899    {
2900        style.front_matter = match v.to_string().as_str() {
2901            "true" => Some(true),
2902            "false" => Some(false),
2903            _ => None,
2904        };
2905    }
2906    if opts.background_color
2907        && let Some(v) = attrs.get_value("background_color")
2908    {
2909        style.background_color = Some(v.to_string());
2910    }
2911    if opts.top_margin
2912        && let Some(v) = attrs.get_value("top_margin")
2913    {
2914        style.top_margin = v.to_string().parse::<i64>().ok();
2915    }
2916    if opts.text_indent
2917        && let Some(v) = attrs.get_value("text_indent")
2918    {
2919        style.text_indent = v.to_string().parse::<i64>().ok();
2920    }
2921    if opts.semantic_role
2922        && let Some(v) = attrs.get_value("semantic_role")
2923    {
2924        style.semantic_role = match v.to_string().as_str() {
2925            "epigraph" => Some(SemanticRole::Epigraph),
2926            // An unknown role is dropped, not guessed at — the same way an unknown
2927            // alignment value above is. A future role read by an older build then
2928            // degrades to a plain blockquote, which is what it looks like anyway.
2929            _ => None,
2930        };
2931    }
2932
2933    style
2934}
2935
2936/// The start a list keeps on its entity, from the number its first item is written with:
2937/// `None` for 1, which every list starts at unless it says otherwise, and a number past what
2938/// every format holds read as the largest it holds (see `list_start`).
2939fn list_start_of(start: u64) -> Option<i64> {
2940    crate::parser_tools::list_start::list_start_from_unsigned(start)
2941}
2942
2943/// Push a finished block into `elements`, applying the djot block-level fields
2944/// plus any round-tripped block-style attributes carried in `style`.
2945#[allow(clippy::too_many_arguments)]
2946fn djot_push_block(
2947    elements: &mut Vec<ParsedElement>,
2948    spans: Vec<ParsedSpan>,
2949    heading_level: Option<i64>,
2950    list_style: Option<ListStyle>,
2951    list_indent: u32,
2952    list_prefix: String,
2953    list_suffix: String,
2954    list_start: Option<i64>,
2955    marker: Option<MarkerType>,
2956    is_code_block: bool,
2957    code_language: Option<String>,
2958    blockquote_depth: u32,
2959    style: DjotBlockStyle,
2960) {
2961    elements.push(ParsedElement::Block(ParsedBlock {
2962        spans,
2963        heading_level,
2964        list_style,
2965        list_indent,
2966        list_prefix,
2967        list_suffix,
2968        list_start,
2969        marker,
2970        is_code_block,
2971        code_language,
2972        blockquote_depth,
2973        blockquote_opens: 0,
2974        line_height: style.line_height,
2975        non_breakable_lines: style.non_breakable_lines,
2976        page_break_before: style.page_break_before,
2977        front_matter: style.front_matter,
2978        direction: style.direction,
2979        background_color: style.background_color,
2980        alignment: style.alignment,
2981        top_margin: style.top_margin,
2982        text_indent: style.text_indent,
2983        semantic_role: style.semantic_role.clone(),
2984    }));
2985}
2986
2987/// A Djot document whose structure cannot be computed, as its source: one plain
2988/// paragraph for each line that holds text, with that line's text as it is written, its
2989/// leading and trailing whitespace aside.
2990///
2991/// A paragraph a line is what keeps the text the same through a save. The Djot writer
2992/// escapes whatever a paragraph opens with and whatever in it would read as markup, so
2993/// the saved text reads back as these same paragraphs. Shown as one paragraph, the
2994/// source kept its line breaks through a save, read back as the same document the parser
2995/// could not take, and every save escaped its backslashes once more. Blank lines are left
2996/// out because the reader keeps no block for an empty paragraph, and a line's outer
2997/// whitespace because it is the source's indentation, which the parser does not read as
2998/// text in a line it can take either.
2999fn djot_source_lines(djot: &str) -> Vec<ParsedElement> {
3000    djot.lines()
3001        .map(|line| line.trim_matches(|c: char| c.is_ascii_whitespace()))
3002        .filter(|line| !line.is_empty())
3003        .map(|line| {
3004            ParsedElement::Block(ParsedBlock {
3005                spans: vec![ParsedSpan {
3006                    text: line.to_string(),
3007                    ..Default::default()
3008                }],
3009                ..Default::default()
3010            })
3011        })
3012        .collect()
3013}
3014
3015/// Parse djot source into the shared [`ParsedElement`] intermediate, mirroring
3016/// [`parse_markdown`]. Uses the [`jotdown`] pull parser.
3017///
3018/// Constructs the document model cannot represent are dropped, and their text
3019/// content is discarded so it never leaks into the document: footnotes, math,
3020/// fenced divs, raw blocks/inline, thematic breaks, description lists,
3021/// captions, symbols, link-reference definitions, and highlight/`mark`. Inline
3022/// images keep their alt text as plain text (the image itself is not modelled),
3023/// matching the Markdown importer. Smart-punctuation events are normalised to
3024/// their canonical Unicode characters so the model→djot→model round-trip is a
3025/// fixpoint.
3026///
3027/// Standalone paragraphs and headings additionally carry the optional
3028/// block-style attributes selected by `options` — paragraph alignment, line
3029/// height, text direction, non-breakable lines and background color — read from
3030/// djot `{key=value}` block attributes (see [`DjotImportOptions`]). List items,
3031/// code blocks and table cells normalise their block styling away.
3032///
3033/// Known model limitations (normalised, not preserved on round-trip):
3034/// table column alignment, and list tight/loose.
3035pub fn parse_djot(djot: &str, options: &DjotImportOptions) -> Vec<ParsedElement> {
3036    use jotdown::{Container as C, Event as E, ListKind};
3037
3038    // `jotdown` recurses with no limit of its own, once per nested container and
3039    // once per line of a paragraph that leaves an inline opener waiting, and a
3040    // stack overflow **aborts the process**: it cannot be caught by
3041    // `catch_unwind`, so there is no recovering from it after the fact and the
3042    // only safe move is not to start. It also panics on a heading deeper than it
3043    // can number, which a caller on an interface thread does not survive either.
3044    // Input is not always the author's own: a project bundle is mailed and
3045    // shared, and an imported document comes from whoever sent it.
3046    //
3047    // `djot_depth::parsable` is the door that keeps the parser within its limits,
3048    // and it degrades rather than refuses, so the signature stays the same and no
3049    // word is lost. Such a heading is escaped into text, as it was written. A
3050    // document nested too deeply by its indentation, which is how a deeply nested
3051    // list is written, is read with the indentation cut: the list comes out
3052    // flattened below a fixed depth, every item and word kept. This used to hand
3053    // back a pasted list nested two hundred levels as one paragraph of markup. A
3054    // long paragraph is parsed on a stack sized for it, and the block holding one
3055    // too long for any stack is shown as its lines, the rest of the document as it
3056    // was.
3057    //
3058    // A document still nested too deeply once flattened, or one the parser panicked
3059    // on, comes back as its lines, one plain paragraph each (see
3060    // `djot_source_lines`). Its structure is not computed, which is the honest
3061    // answer for a document whose structure cannot be computed without ending the
3062    // process, and its text is still there for the writer to see and repair.
3063    let parsable = crate::parser_tools::djot_depth::parsable(djot);
3064    let Some(events) = parsable.as_ref().and_then(|parsable| parsable.events()) else {
3065        return djot_source_lines(djot);
3066    };
3067
3068    let mut elements: Vec<ParsedElement> = Vec::new();
3069    let mut current_spans: Vec<ParsedSpan> = Vec::new();
3070    let mut current_heading: Option<i64> = None;
3071    let mut is_code_block = false;
3072    let mut code_language: Option<String> = None;
3073    let mut blockquote_depth: u32 = 0;
3074    // Where `elements` stood as each open quotation began (see `mark_quotation_opened`).
3075    let mut quotations_open: Vec<usize> = Vec::new();
3076    // Block-style attributes captured from a standalone paragraph/heading's djot
3077    // `{…}` block attributes, consumed when that block is flushed.
3078    let mut pending_style = DjotBlockStyle::default();
3079
3080    // Inline formatting state.
3081    let mut bold = false;
3082    let mut italic = false;
3083    let mut underline = false;
3084    let mut strikeout = false;
3085    let mut code = false;
3086    let mut superscript = false;
3087    let mut subscript = false;
3088    let mut link_href: Option<String> = None;
3089    // Set between an image's Start and End. Its alt text arrives as ordinary
3090    // `Str` events in between, so it has to be captured rather than emitted.
3091    let mut pending_image: Option<ParsedImage> = None;
3092
3093    // List nesting: each entry is (style, prefix, suffix, start); depth = indent + 1.
3094    let mut list_stack: Vec<(ListStyle, String, String, Option<i64>)> = Vec::new();
3095    // Context applied to the next flushed block while inside a list item.
3096    let mut cur_list_style: Option<ListStyle> = None;
3097    let mut cur_list_prefix = String::new();
3098    let mut cur_list_suffix = String::new();
3099    let mut cur_list_start: Option<i64> = None;
3100    let mut cur_list_indent: u32 = 0;
3101    let mut cur_marker: Option<MarkerType> = None;
3102
3103    // Table accumulation.
3104    let mut in_table_cell = false;
3105    let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
3106    let mut current_row: Vec<ParsedTableCell> = Vec::new();
3107    let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
3108    let mut table_header_rows: usize = 0;
3109    let mut row_is_head = false;
3110
3111    // Subtree-skip depth for unrepresentable containers (their entire content
3112    // is dropped). Incremented on the dropped container's `Start` and on every
3113    // nested `Start`; decremented on every `End`.
3114    let mut skip_depth: u32 = 0;
3115
3116    // The label and element index of the footnote definition currently open,
3117    // if any. Djot has no footnote inside a footnote, so one slot is enough
3118    // where the dropped containers need a depth counter.
3119    let mut footnote_open: Option<(String, usize)> = None;
3120
3121    // Push one inline span carrying the current formatting state into the
3122    // active sink (table cell or block). A macro (not a closure) to avoid
3123    // borrowing `current_spans`/`current_cell_spans` across the formatting
3124    // state reads.
3125    macro_rules! push_text {
3126        ($t:expr) => {{
3127            // Alt text belongs to the image, not to the paragraph. While an
3128            // image is open every text event is diverted into its description,
3129            // which is what keeps a photo's caption out of the manuscript's
3130            // word count and out of the search corpus.
3131            if let Some(img) = pending_image.as_mut() {
3132                img.alt.push_str(($t).as_ref());
3133            } else {
3134                let sp = ParsedSpan {
3135                    text: ($t).to_string(),
3136                    bold,
3137                    italic,
3138                    underline,
3139                    strikeout,
3140                    code,
3141                    superscript,
3142                    subscript,
3143                    link_href: link_href.clone(),
3144                    image: None,
3145                    footnote_ref: None,
3146                };
3147                if in_table_cell {
3148                    current_cell_spans.push(sp);
3149                } else {
3150                    current_spans.push(sp);
3151                }
3152            }
3153        }};
3154    }
3155
3156    // Push a completed inline image span into the active sink.
3157    macro_rules! push_image {
3158        ($img:expr) => {{
3159            let sp = ParsedSpan {
3160                text: String::new(),
3161                bold,
3162                italic,
3163                underline,
3164                strikeout,
3165                code,
3166                superscript,
3167                subscript,
3168                link_href: link_href.clone(),
3169                image: Some($img),
3170                footnote_ref: None,
3171            };
3172            if in_table_cell {
3173                current_cell_spans.push(sp);
3174            } else {
3175                current_spans.push(sp);
3176            }
3177        }};
3178    }
3179
3180    // Enter a list item, flushing any unterminated inline content first and
3181    // capturing the list context + task marker for the item's block.
3182    macro_rules! enter_item {
3183        ($marker:expr) => {{
3184            if !current_spans.is_empty() {
3185                djot_push_block(
3186                    &mut elements,
3187                    std::mem::take(&mut current_spans),
3188                    None,
3189                    cur_list_style.clone(),
3190                    cur_list_indent,
3191                    cur_list_prefix.clone(),
3192                    cur_list_suffix.clone(),
3193                    cur_list_start,
3194                    cur_marker.clone(),
3195                    false,
3196                    None,
3197                    blockquote_depth,
3198                    DjotBlockStyle::default(),
3199                );
3200            }
3201            let (style, prefix, suffix, start) = list_stack.last().cloned().unwrap_or((
3202                ListStyle::Disc,
3203                String::new(),
3204                String::new(),
3205                None,
3206            ));
3207            cur_list_style = Some(style);
3208            cur_list_prefix = prefix;
3209            cur_list_suffix = suffix;
3210            cur_list_start = start;
3211            cur_list_indent = list_stack.len().saturating_sub(1) as u32;
3212            cur_marker = $marker;
3213        }};
3214    }
3215
3216    // A hard break: a line break inside the block, [`LINE_SEPARATOR`] in its text, so the
3217    // next line is more of the same paragraph, list item, quotation or heading. A table
3218    // cell is one line, and an image's description is one too, so there it is a space.
3219    macro_rules! line_break {
3220        () => {{
3221            if in_table_cell || pending_image.is_some() {
3222                push_text!(" ");
3223            } else {
3224                push_text!(LINE_SEPARATOR_STR);
3225            }
3226        }};
3227    }
3228
3229    // A line ending inside a code span (see `E::Str` below): a new block, with the list,
3230    // quotation, heading and block style the line was in. A table cell is one line, so
3231    // there it is a space.
3232    //
3233    // The page break is the one style that stays with the first line: it is where the
3234    // block starts, and copied to every line it put each on a page of its own.
3235    macro_rules! split_line {
3236        () => {{
3237            if in_table_cell {
3238                push_text!(" ");
3239            } else if !current_spans.is_empty() && current_heading.is_some() {
3240                // The line before a hard break in a heading is a line of that
3241                // heading, as the line after it is when the heading ends. Read as
3242                // a paragraph, a heading written over two lines came back as a
3243                // paragraph and a heading.
3244                djot_push_block(
3245                    &mut elements,
3246                    std::mem::take(&mut current_spans),
3247                    current_heading,
3248                    None,
3249                    0,
3250                    String::new(),
3251                    String::new(),
3252                    None,
3253                    None,
3254                    false,
3255                    None,
3256                    blockquote_depth,
3257                    pending_style.clone(),
3258                );
3259                pending_style.page_break_before = None;
3260            } else if !current_spans.is_empty() {
3261                djot_push_block(
3262                    &mut elements,
3263                    std::mem::take(&mut current_spans),
3264                    None,
3265                    cur_list_style.clone(),
3266                    cur_list_indent,
3267                    cur_list_prefix.clone(),
3268                    cur_list_suffix.clone(),
3269                    cur_list_start,
3270                    cur_marker.clone(),
3271                    is_code_block,
3272                    code_language.clone(),
3273                    blockquote_depth,
3274                    pending_style.clone(),
3275                );
3276                pending_style.page_break_before = None;
3277            }
3278        }};
3279    }
3280
3281    for event in events {
3282        if skip_depth > 0 {
3283            match event {
3284                E::Start(..) => skip_depth += 1,
3285                E::End(_) => skip_depth -= 1,
3286                _ => {}
3287            }
3288            continue;
3289        }
3290
3291        match event {
3292            // ── Transparent wrappers (unwrap, keep content) ──
3293            E::Start(C::Document, _) | E::End(C::Document) => {}
3294            E::Start(C::Section { .. }, attrs) => {
3295                // A heading's block attributes attach to its enclosing Section,
3296                // not the heading itself; capture them for the heading's flush.
3297                if list_stack.is_empty() {
3298                    pending_style.merge_from(block_attrs_to_style(&attrs, options));
3299                }
3300            }
3301            E::End(C::Section { .. }) => {}
3302            E::Start(C::Div { .. }, _) | E::End(C::Div { .. }) => {}
3303
3304            // ── Blockquote ──
3305            E::Start(C::Blockquote, _) => {
3306                blockquote_depth += 1;
3307                quotations_open.push(elements.len());
3308            }
3309            E::End(C::Blockquote) => {
3310                blockquote_depth = blockquote_depth.saturating_sub(1);
3311                if let Some(first) = quotations_open.pop() {
3312                    mark_quotation_opened(&mut elements, first);
3313                }
3314            }
3315
3316            // ── Lists ──
3317            E::Start(C::List { kind, .. }, _) => {
3318                let (style, prefix, suffix, start) = match kind {
3319                    ListKind::Unordered(b) | ListKind::Task(b) => {
3320                        (djot_bullet_style(b), String::new(), String::new(), None)
3321                    }
3322                    ListKind::Ordered {
3323                        numbering,
3324                        style,
3325                        start,
3326                    } => {
3327                        let (p, s) = djot_ordered_affixes(style);
3328                        (djot_ordered_style(numbering), p, s, list_start_of(start))
3329                    }
3330                };
3331                list_stack.push((style, prefix, suffix, start));
3332            }
3333            E::End(C::List { .. }) => {
3334                list_stack.pop();
3335                cur_list_style = None;
3336                cur_list_start = None;
3337                cur_marker = None;
3338            }
3339            E::Start(C::ListItem, _) => enter_item!(None),
3340            E::Start(C::TaskListItem { checked }, _) => enter_item!(Some(if checked {
3341                MarkerType::Checked
3342            } else {
3343                MarkerType::Unchecked
3344            })),
3345            E::End(C::ListItem) | E::End(C::TaskListItem { .. }) => {
3346                // Tight item without a wrapping paragraph (defensive flush).
3347                if !current_spans.is_empty() {
3348                    djot_push_block(
3349                        &mut elements,
3350                        std::mem::take(&mut current_spans),
3351                        None,
3352                        cur_list_style.clone(),
3353                        cur_list_indent,
3354                        cur_list_prefix.clone(),
3355                        cur_list_suffix.clone(),
3356                        cur_list_start,
3357                        cur_marker.clone(),
3358                        false,
3359                        None,
3360                        blockquote_depth,
3361                        DjotBlockStyle::default(),
3362                    );
3363                }
3364                cur_list_style = None;
3365                cur_list_start = None;
3366                cur_marker = None;
3367            }
3368
3369            // ── Headings, paragraphs, code blocks ──
3370            E::Start(C::Heading { level, .. }, attrs) => {
3371                current_heading = Some(level as i64);
3372                // The block-style attributes live on the enclosing Section;
3373                // merge any placed directly on the heading without clearing them.
3374                pending_style.merge_from(block_attrs_to_style(&attrs, options));
3375            }
3376            E::End(C::Heading { .. }) => {
3377                djot_push_block(
3378                    &mut elements,
3379                    std::mem::take(&mut current_spans),
3380                    current_heading.take(),
3381                    None,
3382                    0,
3383                    String::new(),
3384                    String::new(),
3385                    None,
3386                    None,
3387                    false,
3388                    None,
3389                    blockquote_depth,
3390                    std::mem::take(&mut pending_style),
3391                );
3392            }
3393            E::Start(C::Paragraph, attrs) => {
3394                current_heading = None;
3395                // Block attributes only apply to standalone paragraphs;
3396                // list-item paragraphs normalise their styling away (matching
3397                // the exporter).
3398                pending_style = if list_stack.is_empty() {
3399                    block_attrs_to_style(&attrs, options)
3400                } else {
3401                    DjotBlockStyle::default()
3402                };
3403            }
3404            E::End(C::Paragraph) => {
3405                if !current_spans.is_empty() {
3406                    djot_push_block(
3407                        &mut elements,
3408                        std::mem::take(&mut current_spans),
3409                        None,
3410                        cur_list_style.clone(),
3411                        cur_list_indent,
3412                        cur_list_prefix.clone(),
3413                        cur_list_suffix.clone(),
3414                        cur_list_start,
3415                        cur_marker.clone(),
3416                        false,
3417                        None,
3418                        blockquote_depth,
3419                        std::mem::take(&mut pending_style),
3420                    );
3421                }
3422                cur_list_style = None;
3423                cur_list_start = None;
3424                cur_marker = None;
3425            }
3426            E::Start(C::CodeBlock { language }, _) => {
3427                is_code_block = true;
3428                code_language = if language.is_empty() {
3429                    None
3430                } else {
3431                    Some(language.to_string())
3432                };
3433            }
3434            E::End(C::CodeBlock { .. }) => {
3435                // Strip the single trailing newline jotdown appends.
3436                if let Some(last) = current_spans.last_mut()
3437                    && last.text.ends_with('\n')
3438                {
3439                    last.text.pop();
3440                }
3441                djot_push_block(
3442                    &mut elements,
3443                    std::mem::take(&mut current_spans),
3444                    None,
3445                    None,
3446                    0,
3447                    String::new(),
3448                    String::new(),
3449                    None,
3450                    None,
3451                    true,
3452                    code_language.take(),
3453                    blockquote_depth,
3454                    DjotBlockStyle::default(),
3455                );
3456                is_code_block = false;
3457            }
3458
3459            // ── Tables ──
3460            E::Start(C::Table, _) => {
3461                table_rows.clear();
3462                current_row.clear();
3463                current_cell_spans.clear();
3464                table_header_rows = 0;
3465            }
3466            E::End(C::Table) => {
3467                let own_cells = cells_in_rows(&table_rows);
3468                push_table(
3469                    &mut elements,
3470                    ParsedTable {
3471                        header_rows: table_header_rows,
3472                        rows: std::mem::take(&mut table_rows),
3473                        blockquote_depth,
3474                        blockquote_opens: 0,
3475                    },
3476                    own_cells,
3477                );
3478            }
3479            E::Start(C::TableRow { head }, _) => {
3480                row_is_head = head;
3481                current_row.clear();
3482            }
3483            E::End(C::TableRow { .. }) => {
3484                if row_is_head {
3485                    table_header_rows += 1;
3486                }
3487                table_rows.push(std::mem::take(&mut current_row));
3488            }
3489            E::Start(C::TableCell { .. }, _) => {
3490                in_table_cell = true;
3491                current_cell_spans.clear();
3492            }
3493            E::End(C::TableCell { .. }) => {
3494                in_table_cell = false;
3495                current_row.push(ParsedTableCell {
3496                    spans: std::mem::take(&mut current_cell_spans),
3497                });
3498            }
3499
3500            // ── Inline formatting ──
3501            E::Start(C::Strong, _) => bold = true,
3502            E::End(C::Strong) => bold = false,
3503            E::Start(C::Emphasis, _) => italic = true,
3504            E::End(C::Emphasis) => italic = false,
3505            E::Start(C::Verbatim, _) => code = true,
3506            E::End(C::Verbatim) => code = false,
3507            E::Start(C::Superscript, _) => superscript = true,
3508            E::End(C::Superscript) => superscript = false,
3509            E::Start(C::Subscript, _) => subscript = true,
3510            E::End(C::Subscript) => subscript = false,
3511            E::Start(C::Insert, _) => underline = true,
3512            E::End(C::Insert) => underline = false,
3513            E::Start(C::Delete, _) => strikeout = true,
3514            E::End(C::Delete) => strikeout = false,
3515            // Highlight/mark and bare spans have no model field — keep the text.
3516            E::Start(C::Mark, _) | E::End(C::Mark) => {}
3517            E::Start(C::Span, _) | E::End(C::Span) => {}
3518            E::Start(C::Link(dst, _), _) => link_href = Some(dst.to_string()),
3519            E::End(C::Link(..)) => link_href = None,
3520            // Inline images. Djot writes display size as inline attributes
3521            // (`![alt](src){width=800 height=600}`), which jotdown hands over
3522            // on the `Start` event — verified against jotdown 0.10, including
3523            // quoted values and images mid-sentence.
3524            E::Start(C::Image(src, _), attrs) => {
3525                let attr_num = |key: &str| -> i64 {
3526                    attrs
3527                        .get_value(key)
3528                        .map(|v| v.to_string())
3529                        .and_then(|v| v.trim().parse::<i64>().ok())
3530                        .filter(|n| *n > 0)
3531                        .unwrap_or(0)
3532                };
3533                pending_image = Some(ParsedImage {
3534                    // The writer percent-encodes what the lexer would act on in a
3535                    // source; the source is a name, so it is decoded back.
3536                    src: crate::parser_tools::djot_escape::decode_djot_image_source(&src),
3537                    alt: String::new(),
3538                    width: attr_num("width"),
3539                    height: attr_num("height"),
3540                });
3541            }
3542            E::End(C::Image(..)) => {
3543                if let Some(img) = pending_image.take() {
3544                    push_image!(img);
3545                }
3546            }
3547
3548            // ── Footnote definitions ──
3549            //
3550            // The body is ordinary block content, so it is parsed by the
3551            // ordinary machinery: flush whatever inline run is open, note where
3552            // this definition's blocks start, and let them accumulate. `End`
3553            // lifts them back out. Nesting cannot occur — djot has no footnote
3554            // inside a footnote — so one mark suffices where the dropped
3555            // containers below need a depth counter.
3556            E::Start(C::Footnote { label }, _) => {
3557                if !current_spans.is_empty() {
3558                    djot_push_block(
3559                        &mut elements,
3560                        std::mem::take(&mut current_spans),
3561                        None,
3562                        cur_list_style.clone(),
3563                        cur_list_indent,
3564                        cur_list_prefix.clone(),
3565                        cur_list_suffix.clone(),
3566                        cur_list_start,
3567                        cur_marker.clone(),
3568                        false,
3569                        None,
3570                        blockquote_depth,
3571                        DjotBlockStyle::default(),
3572                    );
3573                }
3574                footnote_open = Some((label.to_string(), elements.len()));
3575            }
3576            E::End(C::Footnote { .. }) => {
3577                if !current_spans.is_empty() {
3578                    djot_push_block(
3579                        &mut elements,
3580                        std::mem::take(&mut current_spans),
3581                        None,
3582                        cur_list_style.clone(),
3583                        cur_list_indent,
3584                        cur_list_prefix.clone(),
3585                        cur_list_suffix.clone(),
3586                        cur_list_start,
3587                        cur_marker.clone(),
3588                        false,
3589                        None,
3590                        blockquote_depth,
3591                        DjotBlockStyle::default(),
3592                    );
3593                }
3594                if let Some((label, start)) = footnote_open.take() {
3595                    let blocks: Vec<ParsedBlock> = elements
3596                        .drain(start..)
3597                        .filter_map(|e| match e {
3598                            ParsedElement::Block(b) => Some(b),
3599                            // A table inside a footnote is not representable as
3600                            // note content; its cells would have to become
3601                            // blocks and lose their structure either way.
3602                            _ => None,
3603                        })
3604                        .collect();
3605                    elements.push(ParsedElement::FootnoteDefinition { label, blocks });
3606                }
3607            }
3608
3609            // ── Unrepresentable containers: drop the entire subtree ──
3610            E::Start(
3611                C::Math { .. }
3612                | C::RawBlock { .. }
3613                | C::RawInline { .. }
3614                | C::DescriptionList
3615                | C::DescriptionDetails
3616                | C::DescriptionTerm
3617                | C::Caption
3618                | C::LinkDefinition { .. },
3619                _,
3620            ) => skip_depth = 1,
3621
3622            // ── Text + atoms ──
3623            // A line ending inside a code span is kept by the parser as a `\n` in the
3624            // span's text. It is read as a new block with the list, quotation or heading
3625            // the line was in, as a `\n` pasted into a block is (see
3626            // `split_block_at_line_breaks`). Kept inside the span, the Djot writer saved
3627            // the code run's lines as blocks of their own, which reloaded as two blocks: a
3628            // document that did not settle into the Djot it saves as. Such a span is what
3629            // an older version wrote for a code run holding a line break, a pasted
3630            // `<pre><code>` passage above all, and reading its lines as blocks is how that
3631            // passage reads in the editor. It is not a hard break: nothing in it asked for
3632            // a line of the same paragraph.
3633            E::Str(s) if code && !is_code_block && pending_image.is_none() && s.contains('\n') => {
3634                for (i, piece) in s.split('\n').enumerate() {
3635                    if i > 0 {
3636                        split_line!();
3637                    }
3638                    if !piece.is_empty() {
3639                        push_text!(piece);
3640                    }
3641                }
3642            }
3643            E::Str(s) => push_text!(s.as_ref()),
3644            E::Softbreak => push_text!(" "),
3645            E::LeftSingleQuote => push_text!("\u{2018}"),
3646            E::RightSingleQuote => push_text!("\u{2019}"),
3647            E::LeftDoubleQuote => push_text!("\u{201C}"),
3648            E::RightDoubleQuote => push_text!("\u{201D}"),
3649            E::Ellipsis => push_text!("\u{2026}"),
3650            E::EnDash => push_text!("\u{2013}"),
3651            E::EmDash => push_text!("\u{2014}"),
3652            E::NonBreakingSpace => push_text!("\u{00A0}"),
3653            E::Hardbreak => line_break!(),
3654            // A footnote reference. jotdown emits this purely syntactically —
3655            // it never checks that a matching `[^label]:` exists anywhere — so a
3656            // reference whose definition lives outside this document (the normal
3657            // state for a host that owns note bodies itself) arrives here just
3658            // the same, and must survive.
3659            E::FootnoteReference(label) => {
3660                let sp = ParsedSpan {
3661                    text: String::new(),
3662                    bold,
3663                    italic,
3664                    underline,
3665                    strikeout,
3666                    code,
3667                    superscript,
3668                    subscript,
3669                    link_href: link_href.clone(),
3670                    image: None,
3671                    footnote_ref: Some(label.to_string()),
3672                };
3673                if in_table_cell {
3674                    current_cell_spans.push(sp);
3675                } else {
3676                    current_spans.push(sp);
3677                }
3678            }
3679            // Symbols, escapes, blanklines, thematic breaks and dangling block
3680            // attributes carry no representable content.
3681            E::Symbol(_) => {}
3682            E::Escape | E::Blankline => {}
3683            E::ThematicBreak(_) | E::Attributes(_) => {}
3684
3685            // Ends of dropped containers (never reached at skip_depth 0) and any
3686            // future variants.
3687            _ => {}
3688        }
3689    }
3690
3691    // Flush any trailing inline content (defensive — Document End closes blocks).
3692    if !current_spans.is_empty() {
3693        djot_push_block(
3694            &mut elements,
3695            std::mem::take(&mut current_spans),
3696            current_heading.take(),
3697            cur_list_style.clone(),
3698            cur_list_indent,
3699            cur_list_prefix.clone(),
3700            cur_list_suffix.clone(),
3701            cur_list_start,
3702            cur_marker.clone(),
3703            is_code_block,
3704            code_language.take(),
3705            blockquote_depth,
3706            std::mem::take(&mut pending_style),
3707        );
3708    }
3709
3710    // An empty document still yields a single empty paragraph (matches
3711    // `parse_markdown`).
3712    if elements.is_empty() {
3713        djot_push_block(
3714            &mut elements,
3715            vec![ParsedSpan {
3716                text: String::new(),
3717                ..Default::default()
3718            }],
3719            None,
3720            None,
3721            0,
3722            String::new(),
3723            String::new(),
3724            None,
3725            None,
3726            false,
3727            None,
3728            0,
3729            DjotBlockStyle::default(),
3730        );
3731    }
3732
3733    elements
3734}
3735
3736#[cfg(test)]
3737mod tests {
3738    use super::*;
3739
3740    /// Helper: flatten parse_markdown output to blocks for tests that don't care about tables.
3741    fn parse_markdown_blocks(md: &str) -> Vec<ParsedBlock> {
3742        ParsedElement::flatten_to_blocks(parse_markdown(md))
3743    }
3744
3745    #[test]
3746    fn test_parse_markdown_simple_paragraph() {
3747        let blocks = parse_markdown_blocks("Hello **world**");
3748        assert_eq!(blocks.len(), 1);
3749        assert!(blocks[0].spans.len() >= 2);
3750        // "Hello " is plain, "world" is bold
3751        let plain_span = blocks[0]
3752            .spans
3753            .iter()
3754            .find(|s| s.text.contains("Hello"))
3755            .unwrap();
3756        assert!(!plain_span.bold);
3757        let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3758        assert!(bold_span.bold);
3759    }
3760
3761    #[test]
3762    fn test_parse_markdown_heading() {
3763        let blocks = parse_markdown_blocks("# Title");
3764        assert_eq!(blocks.len(), 1);
3765        assert_eq!(blocks[0].heading_level, Some(1));
3766        assert_eq!(blocks[0].spans[0].text, "Title");
3767    }
3768
3769    #[test]
3770    fn test_parse_markdown_list() {
3771        let blocks = parse_markdown_blocks("- item1\n- item2");
3772        assert!(blocks.len() >= 2);
3773        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3774        assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
3775    }
3776
3777    /// Helper: extract (is_table, blockquote_depth) per element for nesting assertions.
3778    fn element_depths(elements: &[ParsedElement]) -> Vec<(bool, u32)> {
3779        elements
3780            .iter()
3781            .map(|e| match e {
3782                ParsedElement::Block(b) => (false, b.blockquote_depth),
3783                ParsedElement::Table(t) => (true, t.blockquote_depth),
3784                // Definitions carry no blockquote nesting of their own; this
3785                // helper exists for the nesting assertions and never sees one.
3786                ParsedElement::FootnoteDefinition { .. } => (false, 0),
3787            })
3788            .collect()
3789    }
3790
3791    #[test]
3792    fn test_parse_markdown_table_in_blockquote_records_depth() {
3793        let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3794        assert_eq!(element_depths(&elements), vec![(true, 1)]);
3795    }
3796
3797    #[test]
3798    fn test_parse_markdown_text_then_table_in_blockquote() {
3799        let elements = parse_markdown("> Para\n>\n> | a | b |\n> |---|---|\n> | c | d |");
3800        assert_eq!(element_depths(&elements), vec![(false, 1), (true, 1)]);
3801    }
3802
3803    #[test]
3804    fn test_parse_markdown_table_after_blockquote_closes() {
3805        let elements = parse_markdown("> Para\n\n| a | b |\n|---|---|\n| c | d |");
3806        assert_eq!(element_depths(&elements), vec![(false, 1), (true, 0)]);
3807    }
3808
3809    #[test]
3810    fn test_parse_markdown_table_in_nested_blockquote() {
3811        let elements = parse_markdown(">> | a | b |\n>> |---|---|\n>> | c | d |");
3812        assert_eq!(element_depths(&elements), vec![(true, 2)]);
3813    }
3814
3815    #[test]
3816    fn test_parse_markdown_list_in_blockquote_records_depth() {
3817        let elements = parse_markdown("> - item1\n> - item2");
3818        let depths = element_depths(&elements);
3819        assert_eq!(depths, vec![(false, 1), (false, 1)]);
3820        for e in &elements {
3821            if let ParsedElement::Block(b) = e {
3822                assert_eq!(b.list_style, Some(ListStyle::Disc));
3823            }
3824        }
3825    }
3826
3827    #[test]
3828    fn test_parse_html_table_in_blockquote_records_depth() {
3829        let elements = parse_html_elements(
3830            "<blockquote><table><tr><th>A</th></tr><tr><td>x</td></tr></table></blockquote>",
3831        );
3832        assert_eq!(element_depths(&elements), vec![(true, 1)]);
3833    }
3834
3835    #[test]
3836    fn test_parse_html_table_after_blockquote() {
3837        let elements = parse_html_elements(
3838            "<blockquote><p>Para</p></blockquote><table><tr><td>X</td></tr></table>",
3839        );
3840        let depths = element_depths(&elements);
3841        // The blockquote paragraph carries depth 1; the table is outside (depth 0).
3842        assert!(depths.contains(&(false, 1)), "depths: {depths:?}");
3843        assert!(depths.contains(&(true, 0)), "depths: {depths:?}");
3844    }
3845
3846    #[test]
3847    fn test_flatten_to_blocks_propagates_blockquote_depth() {
3848        let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3849        let blocks = ParsedElement::flatten_to_blocks(elements);
3850        assert!(!blocks.is_empty());
3851        for b in &blocks {
3852            assert_eq!(b.blockquote_depth, 1);
3853        }
3854    }
3855
3856    #[test]
3857    fn test_parse_html_simple() {
3858        let blocks = parse_html("<p>Hello <b>world</b></p>");
3859        assert_eq!(blocks.len(), 1);
3860        assert!(blocks[0].spans.len() >= 2);
3861        let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3862        assert!(bold_span.bold);
3863    }
3864
3865    #[test]
3866    fn test_parse_html_multiple_paragraphs() {
3867        let blocks = parse_html("<p>A</p><p>B</p>");
3868        assert_eq!(blocks.len(), 2);
3869    }
3870
3871    #[test]
3872    fn test_parse_html_heading() {
3873        let blocks = parse_html("<h2>Subtitle</h2>");
3874        assert_eq!(blocks.len(), 1);
3875        assert_eq!(blocks[0].heading_level, Some(2));
3876    }
3877
3878    #[test]
3879    fn test_parse_html_list() {
3880        let blocks = parse_html("<ul><li>one</li><li>two</li></ul>");
3881        assert!(blocks.len() >= 2);
3882        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3883    }
3884
3885    #[test]
3886    fn test_parse_markdown_code_block() {
3887        let blocks = parse_markdown_blocks("```\nfn main() {}\n```");
3888        assert_eq!(blocks.len(), 1);
3889        assert!(blocks[0].is_code_block);
3890        assert!(blocks[0].spans[0].code);
3891        // pulldown-cmark appends a trailing \n to code block text — verify it's stripped
3892        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3893        assert_eq!(
3894            text, "fn main() {}",
3895            "code block text should not have trailing newline"
3896        );
3897    }
3898
3899    #[test]
3900    fn test_parse_markdown_nested_formatting() {
3901        let blocks = parse_markdown_blocks("***bold italic***");
3902        assert_eq!(blocks.len(), 1);
3903        let span = &blocks[0].spans[0];
3904        assert!(span.bold);
3905        assert!(span.italic);
3906    }
3907
3908    #[test]
3909    fn test_parse_markdown_link() {
3910        let blocks = parse_markdown_blocks("[click](http://example.com)");
3911        assert_eq!(blocks.len(), 1);
3912        let span = &blocks[0].spans[0];
3913        assert_eq!(span.text, "click");
3914        assert_eq!(span.link_href, Some("http://example.com".to_string()));
3915    }
3916
3917    #[test]
3918    fn test_parse_markdown_empty() {
3919        let blocks = parse_markdown_blocks("");
3920        assert_eq!(blocks.len(), 1);
3921        assert!(blocks[0].spans[0].text.is_empty());
3922    }
3923
3924    #[test]
3925    fn test_parse_html_empty() {
3926        let blocks = parse_html("");
3927        assert_eq!(blocks.len(), 1);
3928        assert!(blocks[0].spans[0].text.is_empty());
3929    }
3930
3931    #[test]
3932    fn test_parse_html_nested_formatting() {
3933        let blocks = parse_html("<p><b><i>bold italic</i></b></p>");
3934        assert_eq!(blocks.len(), 1);
3935        let span = &blocks[0].spans[0];
3936        assert!(span.bold);
3937        assert!(span.italic);
3938    }
3939
3940    #[test]
3941    fn test_parse_html_link() {
3942        let blocks = parse_html("<p><a href=\"http://example.com\">click</a></p>");
3943        assert_eq!(blocks.len(), 1);
3944        let span = &blocks[0].spans[0];
3945        assert_eq!(span.text, "click");
3946        assert_eq!(span.link_href, Some("http://example.com".to_string()));
3947    }
3948
3949    #[test]
3950    fn test_parse_html_ordered_list() {
3951        let blocks = parse_html("<ol><li>first</li><li>second</li></ol>");
3952        assert!(blocks.len() >= 2);
3953        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3954    }
3955
3956    #[test]
3957    fn test_parse_markdown_ordered_list() {
3958        let blocks = parse_markdown_blocks("1. first\n2. second");
3959        assert!(blocks.len() >= 2);
3960        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3961    }
3962
3963    #[test]
3964    fn test_parse_html_blockquote_nested() {
3965        let blocks = parse_html("<p>before</p><blockquote>quoted</blockquote><p>after</p>");
3966        assert!(blocks.len() >= 3);
3967    }
3968
3969    #[test]
3970    fn test_parse_block_styles_line_height() {
3971        let styles = parse_block_styles("line-height: 1.5");
3972        assert_eq!(styles.line_height, Some(1500));
3973    }
3974
3975    #[test]
3976    fn test_parse_block_styles_direction_rtl() {
3977        let styles = parse_block_styles("direction: rtl");
3978        assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3979    }
3980
3981    #[test]
3982    fn test_parse_block_styles_background_color() {
3983        let styles = parse_block_styles("background-color: #ff0000");
3984        assert_eq!(styles.background_color, Some("#ff0000".to_string()));
3985    }
3986
3987    #[test]
3988    fn test_parse_block_styles_white_space_pre() {
3989        let styles = parse_block_styles("white-space: pre");
3990        assert_eq!(styles.non_breakable_lines, Some(true));
3991    }
3992
3993    #[test]
3994    fn test_parse_block_styles_multiple() {
3995        let styles = parse_block_styles("line-height: 2.0; direction: rtl; background-color: blue");
3996        assert_eq!(styles.line_height, Some(2000));
3997        assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3998        assert_eq!(styles.background_color, Some("blue".to_string()));
3999    }
4000
4001    #[test]
4002    fn test_parse_html_block_styles_extracted() {
4003        let blocks = parse_html(
4004            r#"<p style="line-height: 1.5; direction: rtl; background-color: #ccc">text</p>"#,
4005        );
4006        assert_eq!(blocks.len(), 1);
4007        assert_eq!(blocks[0].line_height, Some(1500));
4008        assert_eq!(blocks[0].direction, Some(TextDirection::RightToLeft));
4009        assert_eq!(blocks[0].background_color, Some("#ccc".to_string()));
4010    }
4011
4012    /// The shape every word processor publishes on the clipboard: a newline
4013    /// after the opening tag, the prose hard-wrapped inside it, and a newline
4014    /// between every pair of blocks. None of it is text.
4015    #[test]
4016    fn test_parse_html_collapses_exporter_line_wrapping() {
4017        let blocks = parse_html(
4018            "<body>\n<p style=\"line-height: 200%\">\nfirst line wrapped\nhere</p>\n\
4019             <p>\nsecond para</p>\n</body>",
4020        );
4021        let texts: Vec<String> = blocks
4022            .iter()
4023            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4024            .collect();
4025        assert_eq!(texts, vec!["first line wrapped here", "second para"]);
4026    }
4027
4028    /// The reported bug: pasting a whole LibreOffice book ended in one block
4029    /// holding every newline the exporter had written between its paragraphs —
4030    /// four thousand blank lines under the last sentence.
4031    #[test]
4032    fn test_parse_html_drops_whitespace_between_blocks() {
4033        let blocks = parse_html("<html>\n<body>\n<p>a</p>\n<p>b</p>\n</body>\n</html>");
4034        assert_eq!(blocks.len(), 2, "no block for the whitespace between them");
4035        assert!(
4036            blocks
4037                .iter()
4038                .all(|b| b.spans.iter().all(|s| !s.text.contains('\n'))),
4039            "no block keeps a literal newline"
4040        );
4041    }
4042
4043    /// A run split across inline elements is still one run, and it collapses to
4044    /// one space rather than disappearing between them.
4045    #[test]
4046    fn test_parse_html_collapses_across_span_boundaries() {
4047        let blocks = parse_html("<p>  <b>bold</b>\n  <i>italic</i>  </p>");
4048        assert_eq!(blocks.len(), 1);
4049        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4050        assert_eq!(text, "bold italic");
4051    }
4052
4053    /// U+00A0 is content, not layout — the space holding "Attention&nbsp;:"
4054    /// together in French must survive a paste.
4055    #[test]
4056    fn test_parse_html_keeps_no_break_space() {
4057        let blocks = parse_html("<p>\nAttention&nbsp;: ici</p>");
4058        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4059        assert_eq!(text, "Attention\u{a0}: ici");
4060    }
4061
4062    /// An empty paragraph is a paragraph. Only the whitespace *between* blocks
4063    /// goes; a block the writer opened stays, however little is in it.
4064    #[test]
4065    fn test_parse_html_keeps_empty_paragraph() {
4066        let blocks = parse_html("<body>\n<p>a</p>\n<p> </p>\n<p>b</p>\n</body>");
4067        assert_eq!(blocks.len(), 3);
4068        let text: String = blocks[1].spans.iter().map(|s| s.text.as_str()).collect();
4069        assert_eq!(text, "");
4070    }
4071
4072    /// `<pre>` and `white-space: pre` mean the source's spacing *is* the
4073    /// content. `nowrap` does not: it forbids wrapping and collapses runs like
4074    /// any other block, which is why it is not in the same test.
4075    #[test]
4076    fn test_parse_html_preserves_whitespace_in_pre() {
4077        // The newline straight after `<pre>` is the one the HTML parser itself
4078        // drops, per spec; every newline after that is content.
4079        let blocks = parse_html("<pre>\nfn main() {\n    let x = 1;\n}</pre>");
4080        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4081        assert_eq!(text, "fn main() {\n    let x = 1;\n}");
4082
4083        // Outside a code block a line break is a new block, and each line keeps its
4084        // spaces.
4085        let styled = parse_html("<p style=\"white-space: pre-wrap\">a\n  b</p>");
4086        let texts: Vec<String> = styled
4087            .iter()
4088            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4089            .collect();
4090        assert_eq!(texts, ["a", "  b"]);
4091
4092        let nowrap = parse_html("<p style=\"white-space: nowrap\">a\n  b</p>");
4093        let text: String = nowrap[0].spans.iter().map(|s| s.text.as_str()).collect();
4094        assert_eq!(text, "a b");
4095    }
4096
4097    /// Table cells are blocks too, and an exporter indents them just as
4098    /// generously as it indents paragraphs.
4099    #[test]
4100    fn test_parse_html_collapses_table_cell_whitespace() {
4101        let elements = parse_html_elements(
4102            "<table>\n<tr>\n<td>\n  one\n</td>\n<td>\n  two\n</td>\n</tr>\n</table>",
4103        );
4104        let table = elements
4105            .iter()
4106            .find_map(|e| match e {
4107                ParsedElement::Table(t) => Some(t),
4108                _ => None,
4109            })
4110            .expect("a table");
4111        let cells: Vec<String> = table.rows[0]
4112            .iter()
4113            .map(|c| c.spans.iter().map(|s| s.text.as_str()).collect())
4114            .collect();
4115        assert_eq!(cells, vec!["one", "two"]);
4116    }
4117
4118    #[test]
4119    fn test_parse_html_white_space_pre() {
4120        let blocks = parse_html(r#"<p style="white-space: pre">code</p>"#);
4121        assert_eq!(blocks.len(), 1);
4122        assert_eq!(blocks[0].non_breakable_lines, Some(true));
4123    }
4124
4125    #[test]
4126    fn test_parse_html_no_styles_returns_none() {
4127        let blocks = parse_html("<p>plain</p>");
4128        assert_eq!(blocks.len(), 1);
4129        assert_eq!(blocks[0].line_height, None);
4130        assert_eq!(blocks[0].direction, None);
4131        assert_eq!(blocks[0].background_color, None);
4132        assert_eq!(blocks[0].non_breakable_lines, None);
4133    }
4134
4135    #[test]
4136    fn test_parse_markdown_nested_list_indent() {
4137        let md = "- top\n  - nested\n    - deep";
4138        let blocks = parse_markdown_blocks(md);
4139        assert_eq!(blocks.len(), 3);
4140        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4141        assert_eq!(blocks[0].list_indent, 0);
4142        assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
4143        assert_eq!(blocks[1].list_indent, 1);
4144        assert_eq!(blocks[2].list_style, Some(ListStyle::Disc));
4145        assert_eq!(blocks[2].list_indent, 2);
4146    }
4147
4148    #[test]
4149    fn test_parse_markdown_nested_ordered_list_indent() {
4150        let md = "1. first\n   1. nested\n   2. nested2";
4151        let blocks = parse_markdown_blocks(md);
4152        assert_eq!(blocks.len(), 3);
4153        assert_eq!(blocks[0].list_indent, 0);
4154        assert_eq!(blocks[1].list_indent, 1);
4155        assert_eq!(blocks[2].list_indent, 1);
4156    }
4157
4158    #[test]
4159    fn test_parse_html_nested_list_indent() {
4160        let html = "<ul><li>top</li><ul><li>nested</li></ul></ul>";
4161        let blocks = parse_html(html);
4162        assert!(blocks.len() >= 2);
4163        assert_eq!(blocks[0].list_indent, 0);
4164        assert_eq!(blocks[1].list_indent, 1);
4165    }
4166
4167    #[test]
4168    fn test_parse_markdown_table() {
4169        let md = "| A | B |\n|---|---|\n| 1 | 2 |";
4170        let elements = parse_markdown(md);
4171        assert_eq!(elements.len(), 1);
4172        match &elements[0] {
4173            ParsedElement::Table(table) => {
4174                assert_eq!(table.header_rows, 1);
4175                assert_eq!(table.rows.len(), 2); // 1 header + 1 body
4176                // Header row
4177                assert_eq!(table.rows[0].len(), 2);
4178                assert_eq!(table.rows[0][0].spans[0].text, "A");
4179                assert_eq!(table.rows[0][1].spans[0].text, "B");
4180                // Body row
4181                assert_eq!(table.rows[1].len(), 2);
4182                assert_eq!(table.rows[1][0].spans[0].text, "1");
4183                assert_eq!(table.rows[1][1].spans[0].text, "2");
4184            }
4185            _ => panic!("Expected ParsedElement::Table"),
4186        }
4187    }
4188
4189    #[test]
4190    fn test_parse_markdown_table_with_formatting() {
4191        let md = "| **bold** | `code` | *italic* |\n|---|---|---|\n| ~~strike~~ | plain | [link](http://x.com) |";
4192        let elements = parse_markdown(md);
4193        assert_eq!(elements.len(), 1);
4194        match &elements[0] {
4195            ParsedElement::Table(table) => {
4196                assert_eq!(table.rows.len(), 2);
4197                // Header: bold cell
4198                assert!(table.rows[0][0].spans[0].bold);
4199                // Header: code cell
4200                assert!(table.rows[0][1].spans[0].code);
4201                // Header: italic cell
4202                assert!(table.rows[0][2].spans[0].italic);
4203                // Body: strikeout cell
4204                assert!(table.rows[1][0].spans[0].strikeout);
4205                // Body: link cell
4206                assert_eq!(
4207                    table.rows[1][2].spans[0].link_href,
4208                    Some("http://x.com".to_string())
4209                );
4210            }
4211            _ => panic!("Expected ParsedElement::Table"),
4212        }
4213    }
4214
4215    #[test]
4216    fn test_parse_markdown_mixed_content_with_table() {
4217        let md = "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter";
4218        let elements = parse_markdown(md);
4219        assert_eq!(elements.len(), 3);
4220        assert!(matches!(&elements[0], ParsedElement::Block(_)));
4221        assert!(matches!(&elements[1], ParsedElement::Table(_)));
4222        assert!(matches!(&elements[2], ParsedElement::Block(_)));
4223    }
4224}
4225
4226#[cfg(test)]
4227mod djot_tests {
4228    use super::*;
4229    use crate::entities::MarkerType;
4230
4231    fn blocks(d: &str) -> Vec<ParsedBlock> {
4232        ParsedElement::flatten_to_blocks(parse_djot(d, &DjotImportOptions::default()))
4233    }
4234
4235    fn first_span_with(b: &ParsedBlock, pred: impl Fn(&ParsedSpan) -> bool) -> &ParsedSpan {
4236        b.spans.iter().find(|s| pred(s)).expect("span not found")
4237    }
4238
4239    #[test]
4240    fn paragraph_bold_italic() {
4241        let b = blocks("normal *bold* _italic_");
4242        assert_eq!(b.len(), 1);
4243        assert!(first_span_with(&b[0], |s| s.text == "bold").bold);
4244        assert!(first_span_with(&b[0], |s| s.text == "italic").italic);
4245    }
4246
4247    #[test]
4248    fn heading_levels() {
4249        assert_eq!(blocks("# H1")[0].heading_level, Some(1));
4250        assert_eq!(blocks("### H3")[0].heading_level, Some(3));
4251        assert_eq!(blocks("###### H6")[0].heading_level, Some(6));
4252    }
4253
4254    #[test]
4255    fn unordered_bullet_styles_are_distinct() {
4256        assert_eq!(blocks("- a")[0].list_style, Some(ListStyle::Disc));
4257        assert_eq!(blocks("* a")[0].list_style, Some(ListStyle::Circle));
4258        assert_eq!(blocks("+ a")[0].list_style, Some(ListStyle::Square));
4259    }
4260
4261    #[test]
4262    fn ordered_delimiters() {
4263        let period = blocks("1. a");
4264        assert_eq!(period[0].list_style, Some(ListStyle::Decimal));
4265        assert_eq!(period[0].list_prefix, "");
4266        assert_eq!(period[0].list_suffix, ".");
4267
4268        let paren = blocks("1) a");
4269        assert_eq!(paren[0].list_suffix, ")");
4270        assert_eq!(paren[0].list_prefix, "");
4271
4272        let paren_paren = blocks("(1) a");
4273        assert_eq!(paren_paren[0].list_prefix, "(");
4274        assert_eq!(paren_paren[0].list_suffix, ")");
4275    }
4276
4277    #[test]
4278    fn task_list_markers() {
4279        let b = blocks("- [ ] a\n- [x] b");
4280        assert_eq!(b.len(), 2);
4281        assert_eq!(b[0].marker, Some(MarkerType::Unchecked));
4282        assert_eq!(b[1].marker, Some(MarkerType::Checked));
4283    }
4284
4285    #[test]
4286    fn code_block_with_language() {
4287        let b = blocks("```rust\nfn main() {}\n```");
4288        assert_eq!(b.len(), 1);
4289        assert!(b[0].is_code_block);
4290        assert_eq!(b[0].code_language.as_deref(), Some("rust"));
4291        let text: String = b[0].spans.iter().map(|s| s.text.as_str()).collect();
4292        assert_eq!(text, "fn main() {}");
4293    }
4294
4295    #[test]
4296    fn link_href() {
4297        let b = blocks("[text](http://example.com)");
4298        let s = first_span_with(&b[0], |s| s.text == "text");
4299        assert_eq!(s.link_href.as_deref(), Some("http://example.com"));
4300    }
4301
4302    #[test]
4303    fn superscript_subscript() {
4304        assert!(first_span_with(&blocks("a^b^")[0], |s| s.text == "b").superscript);
4305        assert!(first_span_with(&blocks("a~b~")[0], |s| s.text == "b").subscript);
4306    }
4307
4308    #[test]
4309    fn delete_insert_verbatim() {
4310        assert!(first_span_with(&blocks("{-x-}")[0], |s| s.text == "x").strikeout);
4311        assert!(first_span_with(&blocks("{+x+}")[0], |s| s.text == "x").underline);
4312        assert!(first_span_with(&blocks("`x`")[0], |s| s.text == "x").code);
4313    }
4314
4315    #[test]
4316    fn blockquote_depth() {
4317        let els = parse_djot("> quoted", &DjotImportOptions::default());
4318        match &els[0] {
4319            ParsedElement::Block(b) => assert_eq!(b.blockquote_depth, 1),
4320            _ => panic!("expected block"),
4321        }
4322    }
4323
4324    #[test]
4325    fn nested_list_indent() {
4326        // Djot nests a sub-list only when a blank line separates it from the
4327        // parent item and it is indented to the parent's content column
4328        // (2 spaces per level). Without the blank line the markers fold into
4329        // the paragraph as lazy continuation.
4330        let b = blocks("- a\n\n  - b\n\n    - c");
4331        assert_eq!(b.len(), 3);
4332        assert_eq!(b[0].list_indent, 0);
4333        assert_eq!(b[1].list_indent, 1);
4334        assert_eq!(b[2].list_indent, 2);
4335    }
4336
4337    #[test]
4338    fn table_parsed_as_table() {
4339        let els = parse_djot(
4340            "| a | b |\n|---|---|\n| c | d |",
4341            &DjotImportOptions::default(),
4342        );
4343        assert_eq!(els.len(), 1);
4344        match &els[0] {
4345            ParsedElement::Table(t) => {
4346                assert_eq!(t.header_rows, 1);
4347                assert_eq!(t.rows.len(), 2);
4348                assert_eq!(t.rows[0][0].spans[0].text, "a");
4349                assert_eq!(t.rows[1][1].spans[0].text, "d");
4350            }
4351            _ => panic!("expected table"),
4352        }
4353    }
4354
4355    #[test]
4356    fn smart_punctuation_normalised_to_unicode() {
4357        let text: String = blocks("a... b---c")[0]
4358            .spans
4359            .iter()
4360            .map(|s| s.text.as_str())
4361            .collect();
4362        assert!(text.contains('\u{2026}'), "ellipsis: {text:?}");
4363        assert!(text.contains('\u{2014}'), "em dash: {text:?}");
4364    }
4365
4366    #[test]
4367    fn unrepresentable_constructs_dropped_without_leaking_text() {
4368        // Thematic break between two paragraphs: no extra block, no stray text.
4369        let b = blocks("para1\n\n---\n\npara2");
4370        assert_eq!(b.len(), 2);
4371        assert_eq!(
4372            b[0].spans
4373                .iter()
4374                .map(|s| s.text.as_str())
4375                .collect::<String>(),
4376            "para1"
4377        );
4378        assert_eq!(
4379            b[1].spans
4380                .iter()
4381                .map(|s| s.text.as_str())
4382                .collect::<String>(),
4383            "para2"
4384        );
4385
4386        // Fenced div is unwrapped: its content survives, the fence does not.
4387        let d = blocks(":::\ninside\n:::");
4388        let joined: String = d
4389            .iter()
4390            .flat_map(|b| b.spans.iter())
4391            .map(|s| s.text.as_str())
4392            .collect();
4393        assert_eq!(joined, "inside");
4394
4395        // Inline math content is dropped, surrounding text kept.
4396        let m = blocks("before $`E=mc^2` after");
4397        let joined: String = m
4398            .iter()
4399            .flat_map(|b| b.spans.iter())
4400            .map(|s| s.text.as_str())
4401            .collect();
4402        assert!(joined.contains("before"), "{joined:?}");
4403        assert!(joined.contains("after"), "{joined:?}");
4404        assert!(!joined.contains("E=mc"), "math leaked: {joined:?}");
4405    }
4406
4407    #[test]
4408    fn empty_document_yields_one_empty_block() {
4409        let b = blocks("");
4410        assert_eq!(b.len(), 1);
4411        assert!(b[0].spans.iter().all(|s| s.text.is_empty()));
4412    }
4413
4414    #[test]
4415    fn block_attributes_parse_into_block() {
4416        let b = blocks(
4417            "{alignment=center line_height=1500 direction=rtl non_breakable_lines=true background_color=\"#ff0000\"}\nhello",
4418        );
4419        assert_eq!(b.len(), 1);
4420        assert_eq!(b[0].alignment, Some(Alignment::Center));
4421        assert_eq!(b[0].line_height, Some(1500));
4422        assert_eq!(b[0].direction, Some(TextDirection::RightToLeft));
4423        assert_eq!(b[0].non_breakable_lines, Some(true));
4424        assert_eq!(b[0].background_color, Some("#ff0000".to_string()));
4425    }
4426
4427    #[test]
4428    fn spacing_block_attributes_parse_into_block() {
4429        // `top_margin` / `text_indent` let one block override the document-wide
4430        // paragraph spacing and first-line indent — what a scene break needs for
4431        // the paragraph that follows it.
4432        let b = blocks("{top_margin=24 text_indent=0}\nhello");
4433        assert_eq!(b.len(), 1);
4434        assert_eq!(b[0].top_margin, Some(24));
4435        assert_eq!(b[0].text_indent, Some(0));
4436    }
4437
4438    #[test]
4439    fn a_zero_text_indent_is_distinct_from_an_absent_one() {
4440        // The whole point of the attribute: `Some(0)` means "explicitly no
4441        // indent", which must not collapse to `None` ("use the document
4442        // default") — otherwise a scene break could not suppress the indent.
4443        let explicit = blocks("{text_indent=0}\nhello");
4444        let absent = blocks("hello");
4445        assert_eq!(explicit[0].text_indent, Some(0));
4446        assert_eq!(absent[0].text_indent, None);
4447        assert!(!explicit[0].is_inline_only());
4448        assert!(absent[0].is_inline_only());
4449    }
4450
4451    #[test]
4452    fn spacing_block_attributes_respect_import_options() {
4453        let src = "{top_margin=24 text_indent=0}\nhello";
4454        let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4455        assert_eq!(b[0].top_margin, None);
4456        assert_eq!(b[0].text_indent, None);
4457    }
4458
4459    #[test]
4460    fn block_attributes_on_heading() {
4461        let b = blocks("{alignment=right}\n# Title");
4462        assert_eq!(b[0].heading_level, Some(1));
4463        assert_eq!(b[0].alignment, Some(Alignment::Right));
4464    }
4465
4466    #[test]
4467    fn block_attributes_respect_import_options() {
4468        // With every optional attribute disabled, the `{…}` block attributes are
4469        // parsed and discarded — only the core paragraph survives.
4470        let src = "{alignment=center line_height=1500}\nhello";
4471        let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4472        assert_eq!(b[0].alignment, None);
4473        assert_eq!(b[0].line_height, None);
4474        assert_eq!(
4475            b[0].spans
4476                .iter()
4477                .map(|s| s.text.as_str())
4478                .collect::<String>(),
4479            "hello"
4480        );
4481    }
4482
4483    #[test]
4484    fn list_item_block_attributes_are_dropped() {
4485        // Block attributes only bind to standalone paragraphs/headings; a list
4486        // item normalises them away (symmetric with the exporter).
4487        let b = blocks("{alignment=center}\n- item");
4488        assert!(b.iter().all(|blk| blk.alignment.is_none()));
4489    }
4490
4491    #[test]
4492    fn unknown_alignment_value_is_ignored() {
4493        let b = blocks("{alignment=sideways}\nhello");
4494        assert_eq!(b[0].alignment, None);
4495    }
4496}
4497
4498// ─── Cheap plain-text extraction ─────────────────────────────────────
4499
4500/// The `U+FFFC OBJECT REPLACEMENT CHARACTER` that stands for a table in the document's
4501/// text.
4502///
4503/// A table is not prose, but it *occupies a position* in the flow: the import mirrors this
4504/// single sentinel into the rope where the table sits
4505/// (`rope_helpers::rope_append_table_anchor`), then the cells as ordinary blocks. Anything
4506/// reconstructing the text a search runs against has to reproduce it, or every offset after
4507/// the first table is short by the two characters (the sentinel and its separator) that the
4508/// document really holds there.
4509pub const TABLE_ANCHOR: &str = "\u{FFFC}";
4510
4511/// `U+2028 LINE SEPARATOR`: a line break inside a block, the one Shift+Enter makes in a
4512/// word processor.
4513///
4514/// A block's text holds it as one character, where it starts a new line of the same
4515/// paragraph, list item, quotation or heading. It is the character Qt's `QTextDocument`
4516/// uses for the same thing. It cannot be `\n`: the rope already uses `\n` between two
4517/// blocks, so a search, an offset or a plain-text export could not tell a line from a
4518/// paragraph.
4519///
4520/// The readers make one of every hard break they meet (Djot's `\` at the end of a line,
4521/// Markdown's, an HTML `<br>` inside a paragraph), and each writer puts it back as its
4522/// format's own line break. A table cell is one line in Djot and Markdown, so there a
4523/// line break is read as a space.
4524pub const LINE_SEPARATOR: char = '\u{2028}';
4525
4526/// [`LINE_SEPARATOR`] as text, for the readers' span builders.
4527const LINE_SEPARATOR_STR: &str = "\u{2028}";
4528
4529/// One span's contribution to the addressable text.
4530///
4531/// An inline object carries no prose but **does** occupy one `U+FFFC` in the
4532/// document (`format_runs_from_spans` mirrors it there), so it has to occupy one
4533/// here too. Leaving it out makes this string shorter than the document it
4534/// claims to be byte-identical to, and every offset past the object — every
4535/// search hit, every comment anchor — lands a character early.
4536fn span_prose(span: &ParsedSpan, out: &mut String) {
4537    if span.image.is_some() || span.footnote_ref.is_some() {
4538        out.push('\u{FFFC}');
4539        return;
4540    }
4541    out.push_str(&span.text);
4542}
4543
4544fn block_prose(block: &ParsedBlock) -> String {
4545    let mut prose = String::new();
4546    for span in &block.spans {
4547        span_prose(span, &mut prose);
4548    }
4549    prose
4550}
4551
4552fn cell_prose(cell: &ParsedTableCell) -> String {
4553    let mut prose = String::new();
4554    for span in &cell.spans {
4555        span_prose(span, &mut prose);
4556    }
4557    prose
4558}
4559
4560/// The prose of a Djot document, with no entities, no store, and no threads.
4561///
4562/// [`parse_djot`] and [`ParsedElement::flatten_to_blocks`] were both already `pub`;
4563/// nothing chained them. This does, and that is the whole trick: it stops at the
4564/// *parse*, where a full import goes on to create a `Block` entity per paragraph, list
4565/// item and table cell, mirror each into the rope, and write its format runs.
4566///
4567/// # Why a project-wide search needs this
4568///
4569/// A host app searching a manuscript must ask "does this scene contain that word" of
4570/// **thousands** of Djot rows, on every keystroke. Doing that by importing each one into
4571/// a document is not a slow feature, it is a frozen app.
4572///
4573/// And searching the Djot *source* instead — the tempting shortcut — is simply wrong:
4574/// the source is markup. `http` matches inside a link's URL, `*` matches an emphasis
4575/// marker, and an occurrence count taken from the source does not agree with what a
4576/// replace re-derives inside the parsed document. Where a replace guards itself with
4577/// "the text moved under me, skip this field", a count taken from markup makes that
4578/// guard fire on perfectly good rows.
4579///
4580/// # The contract
4581///
4582/// The result is **byte-identical to the text the document searches** for the same Djot:
4583/// each block's spans concatenated, blocks joined by a single `\n`, and a table announced
4584/// by its [`TABLE_ANCHOR`] sentinel — exactly the string the import mirrors into the rope
4585/// and that `build_full_text_via_store` recomposes. So an offset found here is an offset
4586/// the document agrees with. On a live document the same string is served by
4587/// `TextDocument::to_addressable_text()`, which reads it off the search plumbing itself.
4588///
4589/// A property in `djot_roundtrip_tests` pins that across the whole generated feature set;
4590/// without it this would be a second, silently-diverging definition of "the text".
4591///
4592/// **Known exception: a footnote *definition*'s body.** This function treats it as out of
4593/// flow and omits it — matching `character_count()`, which does not count it — but the
4594/// live document mirrors its blocks into the rope, so today an in-document search DOES
4595/// run over the note's body and `to_addressable_text()` includes it. The live document
4596/// holds every body after the main text, so this string is the start of that one, and
4597/// every offset of the main text agrees between the two; they differ by the bodies at
4598/// the end. Which side is wrong is an open product question (should search see notes?),
4599/// pinned as current behaviour in
4600/// `addressable_text_tests::footnote_bodies_are_searched_in_the_live_document`.
4601///
4602/// ⚠ It is **not** the same as `TextDocument::to_plain_text()`. That is the human-readable
4603/// *export* — same prose, same order, but with every object anchor omitted, so its offsets
4604/// drift by two characters per preceding table. The authority is what a search sees,
4605/// because that is what a replace edits.
4606/// (`to_plain_text` once also *ordered* a blockquote's prose differently; that was a bug,
4607/// fixed and pinned by `plain_text_order_tests` — see
4608/// `claude_reviews/text-document-plain-text-ordering.md`. The anchors are now the two
4609/// views' only difference.)
4610pub fn djot_to_plain_text(djot: &str, options: &DjotImportOptions) -> String {
4611    // Deliberately NOT `ParsedElement::flatten_to_blocks`: that helper drops a table's
4612    // anchor and yields only its cells, which would silently shift every offset in a
4613    // document containing a table by the two characters the document actually holds there.
4614    let elements = parse_djot(djot, options);
4615
4616    // Sized from the source: the prose is always shorter than the markup that carries it,
4617    // so this allocates once and never grows.
4618    let mut out = String::with_capacity(djot.len());
4619
4620    // The separator is decided by "is this the first block", NOT by "is the output still
4621    // empty". An EMPTY block (an empty code fence, say) is still a block: the document
4622    // holds an empty line for it, and an emptiness test would swallow both the block and
4623    // its separator, shifting every offset after it by one.
4624    let mut first = true;
4625    let push = |text: &str, out: &mut String, first: &mut bool| {
4626        if *first {
4627            *first = false;
4628        } else {
4629            out.push('\n');
4630        }
4631        out.push_str(text);
4632    };
4633
4634    for element in &elements {
4635        match element {
4636            ParsedElement::Block(block) => {
4637                push(&block_prose(block), &mut out, &mut first);
4638            }
4639            // A note's body is **out of flow**: it is not laid out where its
4640            // definition was written, it is not part of a copied fragment, and
4641            // the document does not count its characters. So it is not part of
4642            // the addressable text either — which is the one thing that has to
4643            // stay true here, since `character_count()` and this string are
4644            // compared directly.
4645            //
4646            // Its blocks do live in the rope, because they have to live
4647            // somewhere; they are simply not addressable prose.
4648            ParsedElement::FootnoteDefinition { .. } => {}
4649            ParsedElement::Table(table) => {
4650                // The import mirrors a table into the rope as a lone `U+FFFC` sentinel
4651                // followed by its cells, one per block (`rope_append_table_anchor`). The
4652                // sentinel occupies a real position in the text the document searches, so
4653                // it has to occupy one here too — otherwise every offset after a table is
4654                // short by two characters, and a snippet taken from this string would be
4655                // sliced in the wrong place.
4656                push(TABLE_ANCHOR, &mut out, &mut first);
4657                // A row shorter than the widest holds empty cells in the document, one
4658                // position each: the import completes it (see `ParsedTable::column_count`).
4659                let columns = table.column_count();
4660                for row in &table.rows {
4661                    for column in 0..columns {
4662                        let prose = row.get(column).map(cell_prose).unwrap_or_default();
4663                        push(&prose, &mut out, &mut first);
4664                    }
4665                }
4666            }
4667        }
4668    }
4669    out
4670}
4671
4672#[cfg(test)]
4673mod html_footnote_tests {
4674    use super::*;
4675
4676    /// The reason this exists at all: a producer cannot smuggle a reference
4677    /// through as text, because the Djot escaper neutralises `[`, `^` and `]`.
4678    /// Pinned here so the escaping rule and this workaround cannot drift apart.
4679    #[test]
4680    fn a_literal_reference_in_text_would_be_escaped_into_prose() {
4681        let blocks = parse_html("<p>The ferry.[^1]</p>");
4682        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4683        assert!(
4684            text.contains("[^1]"),
4685            "the parser keeps it as characters: {text:?}"
4686        );
4687        assert!(
4688            blocks[0].spans.iter().all(|s| s.footnote_ref.is_none()),
4689            "text is text — it must not become a reference by accident"
4690        );
4691    }
4692
4693    #[test]
4694    fn an_attributed_element_becomes_a_real_footnote_reference() {
4695        let blocks = parse_html(
4696            r#"<p>The ferry was late.<sup data-footnote-ref="3"></sup> She waited.</p>"#,
4697        );
4698        let refs: Vec<&str> = blocks
4699            .iter()
4700            .flat_map(|b| b.spans.iter())
4701            .filter_map(|s| s.footnote_ref.as_deref())
4702            .collect();
4703        assert_eq!(refs, vec!["3"]);
4704
4705        // And it carries no text of its own, so it cannot be counted as words or
4706        // matched by a search.
4707        let marker = blocks[0]
4708            .spans
4709            .iter()
4710            .find(|s| s.footnote_ref.is_some())
4711            .expect("the reference span");
4712        assert!(marker.text.is_empty(), "{marker:?}");
4713    }
4714
4715    /// The attribute is the contract, not the tag: a producer may use whatever
4716    /// element renders sensibly in a browser.
4717    #[test]
4718    fn any_element_carrying_the_attribute_works() {
4719        for html in [
4720            r#"<p>a<sup data-footnote-ref="1"></sup></p>"#,
4721            r#"<p>a<span data-footnote-ref="1"></span></p>"#,
4722            r##"<p>a<a data-footnote-ref="1" href="#fn1"></a></p>"##,
4723        ] {
4724            let blocks = parse_html(html);
4725            assert_eq!(
4726                blocks
4727                    .iter()
4728                    .flat_map(|b| b.spans.iter())
4729                    .filter_map(|s| s.footnote_ref.as_deref())
4730                    .collect::<Vec<_>>(),
4731                vec!["1"],
4732                "failed for {html}"
4733            );
4734        }
4735    }
4736
4737    #[test]
4738    fn an_empty_or_absent_label_is_not_a_reference() {
4739        for html in [
4740            r#"<p>a<sup data-footnote-ref=""></sup></p>"#,
4741            r#"<p>a<sup></sup></p>"#,
4742        ] {
4743            let blocks = parse_html(html);
4744            assert!(
4745                blocks
4746                    .iter()
4747                    .flat_map(|b| b.spans.iter())
4748                    .all(|s| s.footnote_ref.is_none()),
4749                "failed for {html}"
4750            );
4751        }
4752    }
4753}
4754
4755#[cfg(test)]
4756mod html_depth_tests {
4757    //! HTML nested more deeply than the reader follows, and HTML table cells.
4758
4759    use super::*;
4760
4761    /// The text of each block, in order.
4762    fn block_texts(blocks: &[ParsedBlock]) -> Vec<String> {
4763        blocks
4764            .iter()
4765            .map(|block| block.spans.iter().map(|span| span.text.as_str()).collect())
4766            .collect()
4767    }
4768
4769    /// An HTML list nested past the deepest level an insertion keeps comes in with every
4770    /// item, in its order, the deeper ones side by side at that level. The reader stopped
4771    /// descending at its recursion limit and dropped every item below it (the eighty-sixth
4772    /// level on).
4773    #[test]
4774    fn a_deep_html_list_keeps_every_item_at_the_deepest_level() {
4775        use crate::parser_tools::list_depth::MAX_LIST_INDENT;
4776        let depth = 300;
4777        let mut html: String = (0..depth).map(|i| format!("<ol><li>item {i}")).collect();
4778        html.push_str(&"</li></ol>".repeat(depth));
4779        let blocks = parse_html(&html);
4780        let items: Vec<(String, u32)> = blocks
4781            .iter()
4782            .filter(|block| block.list_style.is_some())
4783            .map(|block| {
4784                let text: String = block.spans.iter().map(|span| span.text.as_str()).collect();
4785                (text, block.list_indent)
4786            })
4787            .collect();
4788        let expected: Vec<(String, u32)> = (0..depth)
4789            .map(|i| (format!("item {i}"), (i as i64).min(MAX_LIST_INDENT) as u32))
4790            .collect();
4791        assert_eq!(items, expected);
4792    }
4793
4794    /// Text outside any item of a list nested too deeply stays, after the items.
4795    #[test]
4796    fn text_in_a_deep_html_list_outside_its_items_is_kept() {
4797        let depth = 40;
4798        let mut html: String = (0..depth).map(|i| format!("<ul><li>item {i}")).collect();
4799        html.push_str("<ul>loose words<li>last</li></ul>");
4800        html.push_str(&"</li></ul>".repeat(depth));
4801        let texts = block_texts(&parse_html(&html)).join(" ");
4802        assert!(texts.contains("loose words"), "{texts:?}");
4803        assert!(texts.contains("last"), "{texts:?}");
4804        assert!(texts.contains("item 39"), "{texts:?}");
4805    }
4806
4807    /// Past the depth the reader follows, it used to keep nothing: the words at the bottom
4808    /// of three hundred nested `<div>`s, or quotations, were dropped from the paste.
4809    #[test]
4810    fn html_nested_past_the_reader_depth_keeps_its_words() {
4811        for (open, close) in [("<div>", "</div>"), ("<blockquote>", "</blockquote>")] {
4812            let html = format!(
4813                "<p>Top.</p>{}<p>Deep <b>words</b></p><p>here.</p>{}<p>End.</p>",
4814                open.repeat(300),
4815                close.repeat(300)
4816            );
4817            let texts = block_texts(&parse_html(&html)).join(" ");
4818            let words: Vec<&str> = texts.split_whitespace().collect();
4819            assert_eq!(
4820                words,
4821                ["Top.", "Deep", "words", "here.", "End."],
4822                "for {open}"
4823            );
4824        }
4825    }
4826
4827    /// The same inside a table cell, read as one paragraph.
4828    #[test]
4829    fn a_table_cell_nested_past_the_reader_depth_keeps_its_words() {
4830        let html = format!(
4831            "<table><tr><td>{}deep words{}</td></tr></table>",
4832            "<span>".repeat(300),
4833            "</span>".repeat(300)
4834        );
4835        let elements = parse_html_elements(&html);
4836        let Some(ParsedElement::Table(table)) = elements.first() else {
4837            panic!("a table: {elements:?}");
4838        };
4839        let text: String = table.rows[0][0]
4840            .spans
4841            .iter()
4842            .map(|span| span.text.as_str())
4843            .collect();
4844        assert_eq!(text, "deep words");
4845    }
4846
4847    /// A cell is one paragraph, so its line breaks and paragraphs are spaces between
4848    /// words. The HTML writer puts a cell's two paragraphs out as `One<br/>Two`, and the
4849    /// reader read them back as the one word `OneTwo`.
4850    #[test]
4851    fn line_breaks_and_paragraphs_in_an_html_cell_separate_words() {
4852        for cell in [
4853            "One<br/>Two",
4854            "<p>One</p><p>Two</p>",
4855            "<ul><li>One</li><li>Two</li></ul>",
4856        ] {
4857            let elements = parse_html_elements(&format!("<table><tr><td>{cell}</td></tr></table>"));
4858            let Some(ParsedElement::Table(table)) = elements.first() else {
4859                panic!("a table: {elements:?}");
4860            };
4861            let text: String = table.rows[0][0]
4862                .spans
4863                .iter()
4864                .map(|span| span.text.as_str())
4865                .collect();
4866            assert_eq!(text, "One Two", "for {cell}");
4867        }
4868    }
4869}