Skip to main content

common/parser_tools/
content_parser.rs

1use crate::entities::{Alignment, ListStyle, MarkerType, SemanticRole, TextDirection};
2use crate::parser_tools::djot_options::DjotImportOptions;
3
4/// An inline image recovered by a parser, before it becomes an `ImageAnchor`.
5///
6/// `src` is whatever the source document pointed at — a relative path, a bare
7/// name, a URL. Parsers do not resolve it; that is the embedding application's
8/// job, and this crate never touches the filesystem.
9#[derive(Debug, Clone, Default, PartialEq, Eq)]
10pub struct ParsedImage {
11    pub src: String,
12    pub alt: String,
13    /// Display size in pixels, `0` when the source did not state one.
14    ///
15    /// Djot and HTML both carry these as attributes (`{width=800}`,
16    /// `<img width=…>`); Markdown has no syntax for them, so a Markdown import
17    /// leaves both zero and the caller supplies intrinsic dimensions.
18    pub width: i64,
19    pub height: i64,
20}
21
22/// A parsed inline span with formatting info.
23///
24/// A span carries *either* text or an image, never both: an image span's `text`
25/// is empty and its description lives in [`ParsedImage::alt`], so alt text
26/// cannot leak into the block's prose (and therefore cannot be counted as
27/// manuscript words or matched by a search).
28#[derive(Debug, Clone, Default)]
29pub struct ParsedSpan {
30    pub text: String,
31    pub bold: bool,
32    pub italic: bool,
33    pub underline: bool,
34    pub strikeout: bool,
35    pub code: bool,
36    /// Superscript (djot `^x^`). Maps to `CharVerticalAlignment::SuperScript`.
37    pub superscript: bool,
38    /// Subscript (djot `~x~`). Maps to `CharVerticalAlignment::SubScript`.
39    pub subscript: bool,
40    pub link_href: Option<String>,
41    /// Set when this span is an inline image rather than text.
42    pub image: Option<ParsedImage>,
43    /// Set when this span is a footnote *reference* rather than text — the
44    /// label naming the note, never the number a reader sees.
45    pub footnote_ref: Option<String>,
46}
47
48/// A parsed table cell containing inline spans.
49#[derive(Debug, Clone)]
50pub struct ParsedTableCell {
51    pub spans: Vec<ParsedSpan>,
52}
53
54/// A parsed table extracted from markdown or HTML.
55#[derive(Debug, Clone)]
56pub struct ParsedTable {
57    /// Number of header rows (typically 1 for markdown tables).
58    pub header_rows: usize,
59    /// All rows (header + body), each containing cells with their inline spans.
60    pub rows: Vec<Vec<ParsedTableCell>>,
61    /// Blockquote nesting depth at the point the table appeared
62    /// (0 = not inside a blockquote), mirroring `ParsedBlock::blockquote_depth`.
63    pub blockquote_depth: u32,
64}
65
66impl ParsedTable {
67    /// How many columns the table needs to hold every cell it has: as many as its widest
68    /// row has cells. A row shorter than that is completed with empty cells by whoever
69    /// builds the table.
70    ///
71    /// The first row does not say it. The rows of a Djot table differ in length when their
72    /// source does, and an HTML table's first row is often one heading cell spanning every
73    /// column: a table sized from its first row kept the cells of a longer row in the model,
74    /// where they showed, while every export writes a row up to the table's width only, so
75    /// the first save dropped them.
76    pub fn column_count(&self) -> usize {
77        self.rows.iter().map(Vec::len).max().unwrap_or(0)
78    }
79}
80
81/// How many cells a table may reach once its short rows are completed (see
82/// [`ParsedTable::column_count`]), however few it has of its own: a table this size loads
83/// in a moment.
84const SQUARED_CELLS_ALWAYS: usize = 4096;
85
86/// How many times the cells it has of its own a larger table may reach once its short
87/// rows are completed.
88///
89/// A table whose rows differ in length because its source wrote them so, a wide heading
90/// row over rows of one cell or a few, grows to about as many times its own cells as it has
91/// columns: a dozen or so for a table a writer makes. The tables that made hundreds of
92/// thousands of cells out of a few kilobytes grow hundreds of times. At four times, a table
93/// of ten columns heading five hundred rows of one cell was read as paragraphs, its grid
94/// lost for a table that loads in a few milliseconds.
95const SQUARED_CELLS_PER_OWN_CELL: usize = 16;
96
97/// Push `table`, which has `own_cells` cells of its own in its source, onto `elements`: as
98/// a table, or as the paragraphs of its cells when completing its short rows would make it
99/// many times that.
100///
101/// Every row of a loaded table is completed to the widest one. A few kilobytes of a table
102/// whose first row is wide and whose other rows hold one cell each, or of HTML cells each
103/// spanning a thousand columns, made hundreds of thousands of empty cells, and held a load
104/// or a paste for seconds. Such a table is read as the paragraphs of its cells that hold
105/// anything, in reading order, at the table's quotation depth: every word stays, the grid
106/// goes.
107fn push_table(elements: &mut Vec<ParsedElement>, table: ParsedTable, own_cells: usize) {
108    let squared = table.rows.len().saturating_mul(table.column_count());
109    let allowed = SQUARED_CELLS_ALWAYS.max(own_cells.saturating_mul(SQUARED_CELLS_PER_OWN_CELL));
110    if squared <= allowed {
111        elements.push(ParsedElement::Table(table));
112        return;
113    }
114    elements.extend(
115        ParsedElement::flatten_to_blocks(vec![ParsedElement::Table(table)])
116            .into_iter()
117            .filter(|block| spans_carry_content(&block.spans))
118            .map(ParsedElement::Block),
119    );
120}
121
122/// The cells a table's rows hold, for [`push_table`].
123fn cells_in_rows(rows: &[Vec<ParsedTableCell>]) -> usize {
124    rows.iter().map(Vec::len).sum()
125}
126
127/// A parsed element: either a block or a table.
128#[derive(Debug, Clone)]
129pub enum ParsedElement {
130    Block(ParsedBlock),
131    Table(ParsedTable),
132    /// A footnote definition: the label its references name, and the blocks
133    /// making up its body.
134    ///
135    /// Separate from `Block` because a definition is not part of the flow it was
136    /// written in — it belongs wherever the output format puts notes, which the
137    /// importer expresses by giving it a detached frame of its own.
138    FootnoteDefinition {
139        label: String,
140        blocks: Vec<ParsedBlock>,
141    },
142}
143
144impl ParsedElement {
145    /// Extract blocks, flattening tables into one block per cell.
146    /// Use when table structure is not needed.
147    pub fn flatten_to_blocks(elements: Vec<ParsedElement>) -> Vec<ParsedBlock> {
148        let mut blocks = Vec::new();
149        for elem in elements {
150            match elem {
151                ParsedElement::Block(b) => blocks.push(b),
152                // A definition is not part of the flow it was written in — it
153                // belongs wherever the format puts notes. Flattening it in
154                // would splice a note's body into the prose at the point the
155                // definition happened to be typed.
156                ParsedElement::FootnoteDefinition { .. } => {}
157                ParsedElement::Table(t) => {
158                    for row in t.rows {
159                        for cell in row {
160                            blocks.push(ParsedBlock {
161                                spans: cell.spans,
162                                heading_level: None,
163                                list_style: None,
164                                list_indent: 0,
165                                list_prefix: String::new(),
166                                list_suffix: String::new(),
167                                marker: None,
168                                is_code_block: false,
169                                code_language: None,
170                                blockquote_depth: t.blockquote_depth,
171                                line_height: None,
172                                non_breakable_lines: None,
173                                page_break_before: None,
174                                direction: None,
175                                background_color: None,
176                                alignment: None,
177                                top_margin: None,
178                                text_indent: None,
179                                semantic_role: None,
180                            });
181                        }
182                    }
183                }
184            }
185        }
186        if blocks.is_empty() {
187            blocks.push(ParsedBlock {
188                spans: vec![ParsedSpan {
189                    text: String::new(),
190                    ..Default::default()
191                }],
192                heading_level: None,
193                list_style: None,
194                list_indent: 0,
195                list_prefix: String::new(),
196                list_suffix: String::new(),
197                marker: None,
198                is_code_block: false,
199                code_language: None,
200                blockquote_depth: 0,
201                line_height: None,
202                non_breakable_lines: None,
203                page_break_before: None,
204                direction: None,
205                background_color: None,
206                alignment: None,
207                top_margin: None,
208                text_indent: None,
209                semantic_role: None,
210            });
211        }
212        blocks
213    }
214}
215
216/// A parsed block (paragraph, heading, list item, code block)
217///
218/// `Default` is an unformatted, empty paragraph — the shape a caller wants when
219/// it has prose and no structure to give it. Deriving it also means a field
220/// added later reaches those callers as "absent" rather than as a compile error
221/// they would fix by guessing a value.
222#[derive(Debug, Clone, Default)]
223pub struct ParsedBlock {
224    pub spans: Vec<ParsedSpan>,
225    pub heading_level: Option<i64>,
226    pub list_style: Option<ListStyle>,
227    pub list_indent: u32,
228    /// Ordered-list delimiter prefix (e.g. `"("` for djot `(1)` lists; empty
229    /// otherwise). Stored on the `List` entity for round-trip fidelity.
230    pub list_prefix: String,
231    /// Ordered-list delimiter suffix (`"."` for `1.`, `")"` for `1)`/`(1)`;
232    /// empty for unordered lists).
233    pub list_suffix: String,
234    /// Task-list checkbox marker (djot `- [ ]` / `- [x]`). Maps to
235    /// `Block.fmt_marker`. `None` for non-task blocks.
236    pub marker: Option<MarkerType>,
237    pub is_code_block: bool,
238    pub code_language: Option<String>,
239    pub blockquote_depth: u32,
240    pub line_height: Option<i64>,
241    pub non_breakable_lines: Option<bool>,
242    /// Start this block on a new page (djot `{page_break_before=true}`). Maps to
243    /// `Block.fmt_page_break_before`. `None` when absent.
244    pub page_break_before: Option<bool>,
245    pub direction: Option<TextDirection>,
246    pub background_color: Option<String>,
247    /// Paragraph alignment (djot `{alignment=left|right|center|justify}`). Maps
248    /// to `Block.fmt_alignment`. `None` when no alignment attribute is present.
249    pub alignment: Option<Alignment>,
250    /// This block's own space-above (djot `{top_margin=<int>}`). Maps to
251    /// `Block.fmt_top_margin` and overrides the document-wide paragraph
252    /// spacing for this block alone. `None` when absent.
253    pub top_margin: Option<i64>,
254    /// This block's own first-line indent (djot `{text_indent=<int>}`). Maps to
255    /// `Block.fmt_text_indent` and overrides the document-wide first-line
256    /// indent for this block alone. `None` when absent.
257    pub text_indent: Option<i64>,
258    /// The enclosing blockquote's semantic role (djot `{semantic_role=epigraph}`),
259    /// written on the quote's first block because block attributes are the only channel
260    /// djot offers. The importer lifts it onto the `Frame`, where it belongs.
261    pub semantic_role: Option<SemanticRole>,
262}
263
264impl ParsedBlock {
265    /// Returns `true` when this block carries no block-level formatting,
266    /// meaning its content is purely inline.
267    pub fn is_inline_only(&self) -> bool {
268        self.heading_level.is_none()
269            && self.list_style.is_none()
270            && !self.is_code_block
271            && self.blockquote_depth == 0
272            && self.line_height.is_none()
273            && self.non_breakable_lines.is_none()
274            && self.page_break_before.is_none()
275            && self.direction.is_none()
276            && self.background_color.is_none()
277            && self.alignment.is_none()
278            && self.top_margin.is_none()
279            && self.text_indent.is_none()
280    }
281}
282
283// ─── Markdown parsing ────────────────────────────────────────────────
284
285/// Labels referenced as `[^label]` in `markdown` with no `[^label]: ...`
286/// definition anywhere in the document.
287///
288/// pulldown-cmark's default footnote mode (`Options::ENABLE_FOOTNOTES`, i.e.
289/// GitHub's syntax) only emits `Event::FootnoteReference` for a label some
290/// `Tag::FootnoteDefinition` actually defines (verified against 0.13) — an
291/// undefined reference silently decomposes into three ordinary `Text` events
292/// (`"["`, `"^label"`, `"]"`) instead, indistinguishable from a reader having
293/// typed literal brackets. `Options::ENABLE_OLD_FOOTNOTES` recognises it, but
294/// as `parse_markdown`'s doc comment explains, trades away multi-paragraph
295/// definition bodies to get there. Neither flag alone gives both, so
296/// `parse_markdown` hands pulldown-cmark a synthetic empty definition for
297/// every such label instead — real enough for it to recognise the reference,
298/// thrown away again before `elements` is returned.
299///
300/// Finding "every such label" is two parts: which labels pulldown-cmark's own
301/// block scanner (a first pass, at the caller's `options`) already recognises
302/// as truly defined, and which `[^…]`-shaped spans exist in the raw text at
303/// all. The second part is a plain scan, not a parser — it can over-match
304/// (inside a code span, inside a fenced block, a coincidental `[^x]:` in the
305/// middle of a sentence that real block parsing would never treat as a
306/// definition), but over-matching only ever produces an unused synthetic
307/// definition: pulldown-cmark's real, correct parse of the augmented text is
308/// what decides whether any span actually becomes a reference, and
309/// `parse_markdown` drops every synthesized definition regardless of whether
310/// that happened.
311fn dangling_footnote_labels(
312    markdown: &str,
313    options: pulldown_cmark::Options,
314) -> std::collections::BTreeSet<String> {
315    use pulldown_cmark::{Event, Parser, Tag};
316
317    let mut defined: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
318    for event in Parser::new_ext(markdown, options) {
319        if let Event::Start(Tag::FootnoteDefinition(label)) = event {
320            defined.insert(label.to_string());
321        }
322    }
323
324    let mut referenced: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
325    let bytes = markdown.as_bytes();
326    let mut search_from = 0usize;
327    while let Some(rel) = markdown[search_from..].find("[^") {
328        let open = search_from + rel;
329        let label_start = open + 2;
330        let Some(close_rel) = markdown[label_start..].find(']') else {
331            break;
332        };
333        let close = label_start + close_rel;
334        let label = &markdown[label_start..close];
335        // `]:` immediately after is definition syntax, not a reference —
336        // pulldown-cmark's own scan above already supplied the ground truth
337        // for which labels are truly defined; this only needs to avoid
338        // counting a definition's own label as a "reference" candidate.
339        let looks_like_definition = bytes.get(close + 1) == Some(&b':');
340        if !label.is_empty() && !looks_like_definition && !label.chars().any(char::is_whitespace) {
341            referenced.insert(label.to_string());
342        }
343        search_from = close + 1;
344    }
345
346    referenced.difference(&defined).cloned().collect()
347}
348
349pub fn parse_markdown(markdown: &str) -> Vec<ParsedElement> {
350    use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
351
352    // ENABLE_FOOTNOTES is required for pulldown-cmark to parse `[^label]`
353    // references and `[^label]: body` definitions at all — without it both
354    // fall through as plain link-reference-shaped text. Deliberately NOT
355    // `ENABLE_OLD_FOOTNOTES` too: that variant makes a reference with no
356    // definition survive (see `dangling_footnote_labels`), but in exchange
357    // breaks multi-paragraph definition bodies — a blank line before an
358    // indented continuation escapes the definition and becomes a sibling
359    // code block, which is exactly the shape this crate's own Markdown
360    // exporter writes for a note with more than one paragraph. Getting both
361    // is `dangling_footnote_labels`'s job, not an `Options` flag's.
362    let options = Options::ENABLE_STRIKETHROUGH
363        | Options::ENABLE_TABLES
364        | Options::ENABLE_TASKLISTS
365        | Options::ENABLE_FOOTNOTES;
366
367    // See `dangling_footnote_labels`: pulldown-cmark's GFM-style footnotes
368    // only recognise `[^label]` as a reference when some definition exists
369    // for it anywhere in the document, so an undefined one — the normal
370    // state for a host that owns note bodies itself — has to be given a
371    // throwaway definition before parsing, or it silently becomes the
372    // literal text "[^label]" instead of surviving as a reference.
373    let dangling = dangling_footnote_labels(markdown, options);
374    let augmented_owner;
375    let source: &str = if dangling.is_empty() {
376        markdown
377    } else {
378        augmented_owner = dangling
379            .iter()
380            .fold(markdown.to_string(), |mut acc, label| {
381                acc.push_str("\n\n[^");
382                acc.push_str(label);
383                acc.push_str("]:\n");
384                acc
385            });
386        &augmented_owner
387    };
388    let parser = Parser::new_ext(source, options);
389
390    let mut elements: Vec<ParsedElement> = Vec::new();
391    let mut current_spans: Vec<ParsedSpan> = Vec::new();
392    let mut current_heading: Option<i64> = None;
393    let mut current_list_style: Option<ListStyle> = None;
394    let mut is_code_block = false;
395    let mut code_language: Option<String> = None;
396    let mut blockquote_depth: u32 = 0;
397    let mut in_block = false;
398
399    // Formatting state stack
400    let mut bold = false;
401    let mut italic = false;
402    let mut strikeout = false;
403    let mut link_href: Option<String> = None;
404    // Set between an image's Start and End; its alt text arrives as Text events.
405    let mut pending_image: Option<ParsedImage> = None;
406
407    // The label and element index of the footnote definition currently open,
408    // if any. Mirrors `parse_djot`'s `footnote_open`: a definition's body is
409    // ordinary block content, parsed by the same machinery as everything
410    // else, then lifted back out at `End` into its own top-level element so
411    // it never becomes part of the flow it was written in. CommonMark
412    // footnote definitions cannot nest, so one slot suffices.
413    let mut footnote_open: Option<(String, usize)> = None;
414
415    // List style stack for nested lists (also tracks nesting depth)
416    let mut list_stack: Vec<Option<ListStyle>> = Vec::new();
417    let mut current_list_indent: u32 = 0;
418
419    // Table tracking state
420    let mut in_table = false;
421    let mut in_table_head = false;
422    let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
423    let mut current_row_cells: Vec<ParsedTableCell> = Vec::new();
424    let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
425    let mut table_header_rows: usize = 0;
426
427    for event in parser {
428        match event {
429            Event::Start(Tag::Paragraph) => {
430                in_block = true;
431                current_heading = None;
432                is_code_block = false;
433            }
434            Event::End(TagEnd::Paragraph) => {
435                if !current_spans.is_empty() || in_block {
436                    elements.push(ParsedElement::Block(ParsedBlock {
437                        spans: std::mem::take(&mut current_spans),
438                        heading_level: current_heading.take(),
439                        list_style: current_list_style.clone(),
440                        list_indent: current_list_indent,
441                        list_prefix: String::new(),
442                        list_suffix: String::new(),
443                        marker: None,
444                        is_code_block: false,
445                        code_language: None,
446                        blockquote_depth,
447                        line_height: None,
448                        non_breakable_lines: None,
449                        page_break_before: None,
450                        direction: None,
451                        background_color: None,
452                        alignment: None,
453                        top_margin: None,
454                        text_indent: None,
455                        semantic_role: None,
456                    }));
457                }
458                in_block = false;
459                current_list_style = None;
460            }
461            Event::Start(Tag::Heading { level, .. }) => {
462                in_block = true;
463                current_heading = Some(heading_level_to_i64(level));
464                is_code_block = false;
465            }
466            Event::End(TagEnd::Heading(_)) => {
467                elements.push(ParsedElement::Block(ParsedBlock {
468                    spans: std::mem::take(&mut current_spans),
469                    heading_level: current_heading.take(),
470                    list_style: None,
471                    list_indent: 0,
472                    list_prefix: String::new(),
473                    list_suffix: String::new(),
474                    marker: None,
475                    is_code_block: false,
476                    code_language: None,
477                    blockquote_depth,
478                    line_height: None,
479                    non_breakable_lines: None,
480                    page_break_before: None,
481                    direction: None,
482                    background_color: None,
483                    alignment: None,
484                    top_margin: None,
485                    text_indent: None,
486                    semantic_role: None,
487                }));
488                in_block = false;
489            }
490            Event::Start(Tag::List(ordered)) => {
491                let style = if ordered.is_some() {
492                    Some(ListStyle::Decimal)
493                } else {
494                    Some(ListStyle::Disc)
495                };
496                list_stack.push(style);
497            }
498            Event::End(TagEnd::List(_)) => {
499                list_stack.pop();
500            }
501            Event::Start(Tag::Item) => {
502                // Flush any accumulated spans from the parent item before
503                // starting a child item in a tight list
504                if !current_spans.is_empty() {
505                    elements.push(ParsedElement::Block(ParsedBlock {
506                        spans: std::mem::take(&mut current_spans),
507                        heading_level: None,
508                        list_style: current_list_style.clone(),
509                        list_indent: current_list_indent,
510                        list_prefix: String::new(),
511                        list_suffix: String::new(),
512                        marker: None,
513                        is_code_block: false,
514                        code_language: None,
515                        blockquote_depth,
516                        line_height: None,
517                        non_breakable_lines: None,
518                        page_break_before: None,
519                        direction: None,
520                        background_color: None,
521                        alignment: None,
522                        top_margin: None,
523                        text_indent: None,
524                        semantic_role: None,
525                    }));
526                }
527                in_block = true;
528                current_list_style = list_stack.last().cloned().flatten();
529                current_list_indent = if list_stack.is_empty() {
530                    0
531                } else {
532                    (list_stack.len() - 1) as u32
533                };
534            }
535            Event::End(TagEnd::Item) => {
536                // The paragraph inside the item will have already been flushed,
537                // but if there was no inner paragraph (tight list), flush now.
538                if !current_spans.is_empty() {
539                    elements.push(ParsedElement::Block(ParsedBlock {
540                        spans: std::mem::take(&mut current_spans),
541                        heading_level: None,
542                        list_style: current_list_style.clone(),
543                        list_indent: current_list_indent,
544                        list_prefix: String::new(),
545                        list_suffix: String::new(),
546                        marker: None,
547                        is_code_block: false,
548                        code_language: None,
549                        blockquote_depth,
550                        line_height: None,
551                        non_breakable_lines: None,
552                        page_break_before: None,
553                        direction: None,
554                        background_color: None,
555                        alignment: None,
556                        top_margin: None,
557                        text_indent: None,
558                        semantic_role: None,
559                    }));
560                }
561                in_block = false;
562                current_list_style = None;
563            }
564            Event::Start(Tag::CodeBlock(kind)) => {
565                in_block = true;
566                is_code_block = true;
567                code_language = match &kind {
568                    pulldown_cmark::CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
569                        Some(lang.to_string())
570                    }
571                    _ => None,
572                };
573            }
574            Event::End(TagEnd::CodeBlock) => {
575                // pulldown-cmark appends a trailing '\n' to code block text — strip it
576                if let Some(last) = current_spans.last_mut()
577                    && last.text.ends_with('\n')
578                {
579                    last.text.truncate(last.text.len() - 1);
580                }
581                elements.push(ParsedElement::Block(ParsedBlock {
582                    spans: std::mem::take(&mut current_spans),
583                    heading_level: None,
584                    list_style: None,
585                    list_indent: 0,
586                    list_prefix: String::new(),
587                    list_suffix: String::new(),
588                    marker: None,
589                    is_code_block: true,
590                    code_language: code_language.take(),
591                    blockquote_depth,
592                    line_height: None,
593                    non_breakable_lines: None,
594                    page_break_before: None,
595                    direction: None,
596                    background_color: None,
597                    alignment: None,
598                    top_margin: None,
599                    text_indent: None,
600                    semantic_role: None,
601                }));
602                in_block = false;
603                is_code_block = false;
604            }
605            // ─── Table events ───────────────────────────────────────
606            Event::Start(Tag::Table(_)) => {
607                in_table = true;
608                in_table_head = false;
609                table_rows.clear();
610                current_row_cells.clear();
611                current_cell_spans.clear();
612                table_header_rows = 0;
613            }
614            Event::End(TagEnd::Table) => {
615                let own_cells = cells_in_rows(&table_rows);
616                push_table(
617                    &mut elements,
618                    ParsedTable {
619                        header_rows: table_header_rows,
620                        rows: std::mem::take(&mut table_rows),
621                        blockquote_depth,
622                    },
623                    own_cells,
624                );
625                in_table = false;
626            }
627            Event::Start(Tag::TableHead) => {
628                in_table_head = true;
629                current_row_cells.clear();
630            }
631            Event::End(TagEnd::TableHead) => {
632                // Flush the header row
633                table_rows.push(std::mem::take(&mut current_row_cells));
634                table_header_rows += 1;
635                in_table_head = false;
636            }
637            Event::Start(Tag::TableRow) => {
638                current_row_cells.clear();
639            }
640            Event::End(TagEnd::TableRow) if !in_table_head => {
641                // Body rows only — header row is flushed in End(TableHead)
642                table_rows.push(std::mem::take(&mut current_row_cells));
643            }
644            Event::Start(Tag::TableCell) => {
645                current_cell_spans.clear();
646            }
647            Event::End(TagEnd::TableCell) => {
648                current_row_cells.push(ParsedTableCell {
649                    spans: std::mem::take(&mut current_cell_spans),
650                });
651            }
652            // ─── Inline formatting ──────────────────────────────────
653            Event::Start(Tag::Emphasis) => {
654                italic = true;
655            }
656            Event::End(TagEnd::Emphasis) => {
657                italic = false;
658            }
659            Event::Start(Tag::Strong) => {
660                bold = true;
661            }
662            Event::End(TagEnd::Strong) => {
663                bold = false;
664            }
665            Event::Start(Tag::Strikethrough) => {
666                strikeout = true;
667            }
668            Event::End(TagEnd::Strikethrough) => {
669                strikeout = false;
670            }
671            Event::Start(Tag::Link { dest_url, .. }) => {
672                link_href = Some(dest_url.to_string());
673            }
674            Event::End(TagEnd::Link) => {
675                link_href = None;
676            }
677            // Markdown has no syntax for display size, so width/height stay 0
678            // and the caller supplies the image's intrinsic dimensions.
679            Event::Start(Tag::Image { dest_url, .. }) => {
680                pending_image = Some(ParsedImage {
681                    src: dest_url.to_string(),
682                    alt: String::new(),
683                    width: 0,
684                    height: 0,
685                });
686            }
687            Event::End(TagEnd::Image) => {
688                if let Some(image) = pending_image.take() {
689                    let span = ParsedSpan {
690                        text: String::new(),
691                        bold,
692                        italic,
693                        underline: false,
694                        strikeout,
695                        code: false,
696                        superscript: false,
697                        subscript: false,
698                        link_href: link_href.clone(),
699                        image: Some(image),
700                        footnote_ref: None,
701                    };
702                    if in_table {
703                        current_cell_spans.push(span);
704                    } else {
705                        if !in_block {
706                            in_block = true;
707                        }
708                        current_spans.push(span);
709                    }
710                }
711            }
712            Event::Text(text) => {
713                // Inside an image this is its alt text, which pulldown-cmark
714                // emits as an ordinary Text event. Without this guard it fell
715                // through and landed in the paragraph as prose — the image was
716                // dropped and its description silently became manuscript text.
717                if let Some(img) = pending_image.as_mut() {
718                    img.alt.push_str(&text);
719                    continue;
720                }
721                let span = ParsedSpan {
722                    text: text.to_string(),
723                    bold,
724                    italic,
725                    underline: false,
726                    strikeout,
727                    code: is_code_block,
728                    superscript: false,
729                    subscript: false,
730                    link_href: link_href.clone(),
731                    image: None,
732                    footnote_ref: None,
733                };
734                if in_table {
735                    current_cell_spans.push(span);
736                } else {
737                    if !in_block {
738                        in_block = true;
739                    }
740                    current_spans.push(span);
741                }
742            }
743            Event::Code(text) => {
744                let span = ParsedSpan {
745                    text: text.to_string(),
746                    bold,
747                    italic,
748                    underline: false,
749                    strikeout,
750                    code: true,
751                    superscript: false,
752                    subscript: false,
753                    link_href: link_href.clone(),
754                    image: None,
755                    footnote_ref: None,
756                };
757                if in_table {
758                    current_cell_spans.push(span);
759                } else {
760                    if !in_block {
761                        in_block = true;
762                    }
763                    current_spans.push(span);
764                }
765            }
766            Event::SoftBreak => {
767                let span = ParsedSpan {
768                    text: " ".to_string(),
769                    bold,
770                    italic,
771                    underline: false,
772                    strikeout,
773                    code: false,
774                    superscript: false,
775                    subscript: false,
776                    link_href: link_href.clone(),
777                    image: None,
778                    footnote_ref: None,
779                };
780                if in_table {
781                    current_cell_spans.push(span);
782                } else {
783                    current_spans.push(span);
784                }
785            }
786            Event::HardBreak if !current_spans.is_empty() || in_block => {
787                // Finalize current block
788                elements.push(ParsedElement::Block(ParsedBlock {
789                    spans: std::mem::take(&mut current_spans),
790                    heading_level: current_heading.take(),
791                    list_style: current_list_style.clone(),
792                    list_indent: current_list_indent,
793                    list_prefix: String::new(),
794                    list_suffix: String::new(),
795                    marker: None,
796                    is_code_block,
797                    code_language: code_language.clone(),
798                    blockquote_depth,
799                    line_height: None,
800                    non_breakable_lines: None,
801                    page_break_before: None,
802                    direction: None,
803                    background_color: None,
804                    alignment: None,
805                    top_margin: None,
806                    text_indent: None,
807                    semantic_role: None,
808                }));
809            }
810            Event::Start(Tag::BlockQuote(_)) => {
811                blockquote_depth += 1;
812            }
813            Event::End(TagEnd::BlockQuote(_)) => {
814                blockquote_depth = blockquote_depth.saturating_sub(1);
815            }
816            // ── Footnote definitions ──
817            //
818            // `[^label]: body` opens a block-level container; its body is
819            // ordinary paragraphs/lists/etc., pushed onto `elements` by the
820            // ordinary machinery above. `End` drains everything pushed since
821            // `Start` back out into one `FootnoteDefinition`, exactly as
822            // `parse_djot` does for the same `[^label]: body` syntax.
823            // Flushing any spans left open by an interrupted block first
824            // matches every other container boundary in this parser
825            // (List/Item/BlockQuote).
826            Event::Start(Tag::FootnoteDefinition(label)) => {
827                if !current_spans.is_empty() {
828                    elements.push(ParsedElement::Block(ParsedBlock {
829                        spans: std::mem::take(&mut current_spans),
830                        heading_level: current_heading.take(),
831                        list_style: current_list_style.clone(),
832                        list_indent: current_list_indent,
833                        list_prefix: String::new(),
834                        list_suffix: String::new(),
835                        marker: None,
836                        is_code_block: false,
837                        code_language: None,
838                        blockquote_depth,
839                        line_height: None,
840                        non_breakable_lines: None,
841                        page_break_before: None,
842                        direction: None,
843                        background_color: None,
844                        alignment: None,
845                        top_margin: None,
846                        text_indent: None,
847                        semantic_role: None,
848                    }));
849                }
850                footnote_open = Some((label.to_string(), elements.len()));
851            }
852            Event::End(TagEnd::FootnoteDefinition) => {
853                if !current_spans.is_empty() {
854                    elements.push(ParsedElement::Block(ParsedBlock {
855                        spans: std::mem::take(&mut current_spans),
856                        heading_level: current_heading.take(),
857                        list_style: current_list_style.clone(),
858                        list_indent: current_list_indent,
859                        list_prefix: String::new(),
860                        list_suffix: String::new(),
861                        marker: None,
862                        is_code_block: false,
863                        code_language: None,
864                        blockquote_depth,
865                        line_height: None,
866                        non_breakable_lines: None,
867                        page_break_before: None,
868                        direction: None,
869                        background_color: None,
870                        alignment: None,
871                        top_margin: None,
872                        text_indent: None,
873                        semantic_role: None,
874                    }));
875                }
876                if let Some((label, start)) = footnote_open.take() {
877                    let blocks: Vec<ParsedBlock> = elements
878                        .drain(start..)
879                        .filter_map(|e| match e {
880                            ParsedElement::Block(b) => Some(b),
881                            // A table inside a footnote is not representable
882                            // as note content, same limitation as djot.
883                            _ => None,
884                        })
885                        .collect();
886                    elements.push(ParsedElement::FootnoteDefinition { label, blocks });
887                }
888            }
889            // A footnote reference. pulldown-cmark emits this whether or not
890            // a matching `[^label]:` definition exists anywhere in the
891            // document — a dangling reference (the normal state for a host
892            // that owns note bodies itself) must survive just the same,
893            // mirroring `parse_djot`'s `E::FootnoteReference` handling.
894            Event::FootnoteReference(label) => {
895                let span = ParsedSpan {
896                    text: String::new(),
897                    bold,
898                    italic,
899                    underline: false,
900                    strikeout,
901                    code: false,
902                    superscript: false,
903                    subscript: false,
904                    link_href: link_href.clone(),
905                    image: None,
906                    footnote_ref: Some(label.to_string()),
907                };
908                if in_table {
909                    current_cell_spans.push(span);
910                } else {
911                    if !in_block {
912                        in_block = true;
913                    }
914                    current_spans.push(span);
915                }
916            }
917            _ => {}
918        }
919    }
920
921    // Flush any remaining content
922    if !current_spans.is_empty() {
923        elements.push(ParsedElement::Block(ParsedBlock {
924            spans: std::mem::take(&mut current_spans),
925            heading_level: current_heading,
926            list_style: current_list_style,
927            list_indent: current_list_indent,
928            list_prefix: String::new(),
929            list_suffix: String::new(),
930            marker: None,
931            is_code_block,
932            code_language: code_language.take(),
933            blockquote_depth,
934            line_height: None,
935            non_breakable_lines: None,
936            page_break_before: None,
937            direction: None,
938            background_color: None,
939            alignment: None,
940            top_margin: None,
941            text_indent: None,
942            semantic_role: None,
943        }));
944    }
945
946    // Drop the throwaway definitions `dangling_footnote_labels` asked for —
947    // they exist only to make pulldown-cmark recognise the reference as
948    // real, never to become a note the document owns. `elements` cannot
949    // become empty from this: a label reached `dangling` only via a
950    // reference actually present in `markdown`, so its surrounding block
951    // survives even with the synthetic definition gone.
952    if !dangling.is_empty() {
953        elements.retain(
954            |e| !matches!(e, ParsedElement::FootnoteDefinition { label, .. } if dangling.contains(label)),
955        );
956    }
957
958    // If no elements were parsed, create a single empty paragraph
959    if elements.is_empty() {
960        elements.push(ParsedElement::Block(ParsedBlock {
961            spans: vec![ParsedSpan {
962                text: String::new(),
963                ..Default::default()
964            }],
965            heading_level: None,
966            list_style: None,
967            list_indent: 0,
968            list_prefix: String::new(),
969            list_suffix: String::new(),
970            marker: None,
971            is_code_block: false,
972            code_language: None,
973            blockquote_depth: 0,
974            line_height: None,
975            non_breakable_lines: None,
976            page_break_before: None,
977            direction: None,
978            background_color: None,
979            alignment: None,
980            top_margin: None,
981            text_indent: None,
982            semantic_role: None,
983        }));
984    }
985
986    elements
987}
988
989fn heading_level_to_i64(level: pulldown_cmark::HeadingLevel) -> i64 {
990    use pulldown_cmark::HeadingLevel;
991    match level {
992        HeadingLevel::H1 => 1,
993        HeadingLevel::H2 => 2,
994        HeadingLevel::H3 => 3,
995        HeadingLevel::H4 => 4,
996        HeadingLevel::H5 => 5,
997        HeadingLevel::H6 => 6,
998    }
999}
1000
1001// ─── HTML parsing ────────────────────────────────────────────────────
1002
1003use scraper::Node;
1004
1005/// Parsed CSS block-level styles from an inline `style` attribute.
1006#[derive(Debug, Clone, Default)]
1007struct BlockStyles {
1008    line_height: Option<i64>,
1009    non_breakable_lines: Option<bool>,
1010    page_break_before: Option<bool>,
1011    direction: Option<TextDirection>,
1012    background_color: Option<String>,
1013    /// `white-space` asks for the source's own spacing to be kept verbatim.
1014    /// Separate from `non_breakable_lines` because the two do not coincide:
1015    /// `nowrap` forbids wrapping but still collapses runs of spaces, so a
1016    /// block carrying it must be normalised like any other.
1017    preserve_whitespace: Option<bool>,
1018}
1019
1020/// Parse relevant CSS properties from an inline style string.
1021/// Handles: line-height, white-space, break-before/page-break-before, direction,
1022/// background-color.
1023fn parse_block_styles(style: &str) -> BlockStyles {
1024    let mut result = BlockStyles::default();
1025    for part in style.split(';') {
1026        let part = part.trim();
1027        if let Some((prop, val)) = part.split_once(':') {
1028            let prop = prop.trim().to_ascii_lowercase();
1029            let val = val.trim();
1030            match prop.as_str() {
1031                "line-height" => {
1032                    // Try parsing as a plain number (multiplier)
1033                    if let Ok(v) = val.parse::<f64>() {
1034                        result.line_height = Some((v * 1000.0) as i64);
1035                    }
1036                }
1037                "white-space" if val == "pre" || val == "nowrap" || val == "pre-wrap" => {
1038                    result.non_breakable_lines = Some(true);
1039                    if val != "nowrap" {
1040                        result.preserve_whitespace = Some(true);
1041                    }
1042                }
1043                // CSS3 `break-before` and its CSS2 predecessor `page-break-before` mean
1044                // the same thing; both are read because both are written (browsers still
1045                // want the legacy spelling, and so do most EPUB engines).
1046                "break-before" | "page-break-before" => {
1047                    result.page_break_before = match val.to_ascii_lowercase().as_str() {
1048                        "page" | "always" | "left" | "right" | "recto" | "verso" => Some(true),
1049                        "avoid" | "auto" => Some(false),
1050                        _ => None,
1051                    };
1052                }
1053                "direction" => {
1054                    if val.eq_ignore_ascii_case("rtl") {
1055                        result.direction = Some(TextDirection::RightToLeft);
1056                    } else if val.eq_ignore_ascii_case("ltr") {
1057                        result.direction = Some(TextDirection::LeftToRight);
1058                    }
1059                }
1060                "background-color" | "background" => {
1061                    result.background_color = Some(val.to_string());
1062                }
1063                _ => {}
1064            }
1065        }
1066    }
1067    result
1068}
1069
1070/// The five characters HTML calls whitespace.
1071///
1072/// Deliberately not `char::is_whitespace`, which is `true` for U+00A0: a
1073/// no-break space is a character the writer typed — the one French typography
1074/// puts before a colon, holding "Attention&nbsp;:" together — and collapsing it
1075/// away is a silent edit of the text, not of its layout.
1076fn is_html_space(ch: char) -> bool {
1077    matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{0C}')
1078}
1079
1080/// Whether a span run holds anything a reader would see.
1081fn spans_carry_content(spans: &[ParsedSpan]) -> bool {
1082    spans
1083        .iter()
1084        .any(|s| !s.text.is_empty() || s.image.is_some() || s.footnote_ref.is_some())
1085}
1086
1087/// Apply the CSS `white-space: normal` rules to one block's inline spans: every
1088/// run of HTML whitespace collapses to a single space, and the runs at the
1089/// block's two ends disappear entirely.
1090///
1091/// Per block rather than per text node, because a run crosses span boundaries —
1092/// `<b>word</b>\n<i>next</i>` is one collapsible run split over three nodes —
1093/// and because only the block knows where its own ends are.
1094///
1095/// Without this, an exporter's own source formatting reaches the document as
1096/// text. LibreOffice writes a newline after every `<p …>` and hard-wraps the
1097/// prose inside it at about seventy columns, so pasting a whole book put a
1098/// blank line before every paragraph and a break through the middle of every
1099/// sentence.
1100fn collapse_inline_whitespace(spans: &mut [ParsedSpan]) {
1101    fn is_replaced(span: &ParsedSpan) -> bool {
1102        span.image.is_some() || span.footnote_ref.is_some()
1103    }
1104
1105    // Collapse within each span first. Nothing non-empty is emptied here, which
1106    // is what lets the second pass read `is_empty()` as "this span held no text
1107    // to begin with" — the marker a `<br>` leaves behind.
1108    for span in spans.iter_mut() {
1109        if is_replaced(span) {
1110            continue;
1111        }
1112        let mut out = String::with_capacity(span.text.len());
1113        let mut in_run = false;
1114        for ch in span.text.chars() {
1115            if is_html_space(ch) {
1116                if !in_run {
1117                    out.push(' ');
1118                    in_run = true;
1119                }
1120            } else {
1121                out.push(ch);
1122                in_run = false;
1123            }
1124        }
1125        span.text = out;
1126    }
1127
1128    // The block's start counts as "already ended on a space", so a run opening
1129    // the block is dropped outright.
1130    let mut prev_ends_with_space = true;
1131    for span in spans.iter_mut() {
1132        if is_replaced(span) {
1133            prev_ends_with_space = false;
1134            continue;
1135        }
1136        if span.text.is_empty() {
1137            // The empty span a `<br>` leaves. It contributes no text, but it
1138            // still ends the collapsible run either side of it — without that,
1139            // `x <br> y` closes up into "xy".
1140            prev_ends_with_space = false;
1141            continue;
1142        }
1143        if prev_ends_with_space && span.text.starts_with(' ') {
1144            span.text.remove(0);
1145        }
1146        // A span emptied by that drop leaves the run open rather than closing
1147        // it: the space it carried belongs to the run its neighbours share.
1148        if !span.text.is_empty() {
1149            prev_ends_with_space = span.text.ends_with(' ');
1150        }
1151    }
1152
1153    // The block's trailing run, in whichever span still holds it. Replaced
1154    // content stops the walk: a space before a trailing image is between two
1155    // pieces of content, not after the last one.
1156    for span in spans.iter_mut().rev() {
1157        if is_replaced(span) {
1158            break;
1159        }
1160        if span.text.is_empty() {
1161            continue;
1162        }
1163        if span.text.ends_with(' ') {
1164            span.text.pop();
1165        }
1166        break;
1167    }
1168}
1169
1170/// `block` as one block per line of its text.
1171///
1172/// The model has no line break inside a block: a new line is a new block, which is
1173/// what the Djot reader makes of a hard break and what a plain-text paste makes of a
1174/// line break. Text that kept its source's line breaks (`<pre>`, `white-space: pre`)
1175/// held them inside one block, where the editor showed them as lines and the Djot
1176/// writer had nothing to write them as. Written raw, a blank line ended the paragraph
1177/// and the lines after it were read back as the lists, headings and quotations their
1178/// markers spelled, however deep.
1179///
1180/// Each line keeps the block's own formatting and its spans' formatting, but for the
1181/// page break, which stays with the first line: it is where the block starts, and on
1182/// every line it put each on a page of its own. An empty line is an empty block. The
1183/// line break that ends the last line starts no line of its own, as a browser shows it.
1184/// A block without a line break is returned as it is.
1185///
1186/// Every line is built from copies of the block and of the span it comes from that
1187/// hold no text, so the split costs in proportion to the text: built from whole copies,
1188/// each line copied the whole text again, and a paste of 64,000 preformatted lines
1189/// took seconds.
1190pub fn split_block_at_line_breaks(mut block: ParsedBlock) -> Vec<ParsedBlock> {
1191    let breaks = |span: &ParsedSpan| span.image.is_none() && span.text.contains('\n');
1192    if !block.spans.iter().any(breaks) {
1193        return vec![block];
1194    }
1195    let mut lines: Vec<Vec<ParsedSpan>> = vec![Vec::new()];
1196    for mut span in std::mem::take(&mut block.spans) {
1197        if !breaks(&span) {
1198            push_to_last_line(&mut lines, span);
1199            continue;
1200        }
1201        let text = std::mem::take(&mut span.text);
1202        for (i, piece) in text.split('\n').enumerate() {
1203            if i > 0 {
1204                lines.push(Vec::new());
1205            }
1206            if !piece.is_empty() {
1207                push_to_last_line(
1208                    &mut lines,
1209                    ParsedSpan {
1210                        text: piece.to_string(),
1211                        ..span.clone()
1212                    },
1213                );
1214            }
1215        }
1216    }
1217    if lines.len() > 1 && lines.last().is_some_and(Vec::is_empty) {
1218        lines.pop();
1219    }
1220    let later_line = ParsedBlock {
1221        page_break_before: None,
1222        ..block.clone()
1223    };
1224    let mut first_line = Some(block);
1225    lines
1226        .into_iter()
1227        .map(|spans| {
1228            let spans = if spans.is_empty() {
1229                vec![ParsedSpan::default()]
1230            } else {
1231                spans
1232            };
1233            match first_line.take() {
1234                Some(first) => ParsedBlock { spans, ..first },
1235                None => ParsedBlock {
1236                    spans,
1237                    ..later_line.clone()
1238                },
1239            }
1240        })
1241        .collect()
1242}
1243
1244/// Append `span` to the last of `lines`, which [`split_block_at_line_breaks`] keeps
1245/// non-empty.
1246fn push_to_last_line(lines: &mut Vec<Vec<ParsedSpan>>, span: ParsedSpan) {
1247    match lines.last_mut() {
1248        Some(line) => line.push(span),
1249        None => lines.push(vec![span]),
1250    }
1251}
1252
1253pub fn parse_html(html: &str) -> Vec<ParsedBlock> {
1254    ParsedElement::flatten_to_blocks(parse_html_elements(html))
1255}
1256
1257/// The attribute an HTML producer uses to say "this element *is* a footnote
1258/// reference", and the label it names.
1259///
1260/// A data attribute on an otherwise-ordinary element, rather than a literal
1261/// `[^label]` in the text, because the text a producer emits is *escaped* on the
1262/// way to Djot — [`escape_djot_inline`](crate::parser_tools::djot_escape::escape_djot_inline)
1263/// neutralises `[`, `]` and `^`, so a reference smuggled through as characters
1264/// arrives as `\[\^1\]` and is prose rather than a reference. It has to be a
1265/// node before it is serialised, which is what this makes it.
1266///
1267/// [`ParsedSpan::footnote_ref`] is the same field the Markdown/Djot reader
1268/// already fills, so everything downstream — the document model, the Djot
1269/// writer, numbering — needs no change at all.
1270pub const HTML_FOOTNOTE_ATTR: &str = "data-footnote-ref";
1271
1272/// Build a footnote-reference span from any element carrying
1273/// [`HTML_FOOTNOTE_ATTR`].
1274fn html_footnote_span(
1275    el: &scraper::node::Element,
1276    link_href: Option<String>,
1277) -> Option<ParsedSpan> {
1278    let label = el.attr(HTML_FOOTNOTE_ATTR)?.trim();
1279    if label.is_empty() {
1280        return None;
1281    }
1282    Some(ParsedSpan {
1283        text: String::new(),
1284        link_href,
1285        footnote_ref: Some(label.to_string()),
1286        ..Default::default()
1287    })
1288}
1289
1290/// Build an inline-image span from an `<img>` element, if it has a usable
1291/// source.
1292///
1293/// `<img>` was matched by none of the HTML walker's three tag dispatches, so it
1294/// fell into their wildcard arms and was dropped whole — silently, and unlike
1295/// Markdown not even leaving its alt text behind. That is the path a browser or
1296/// Word paste travels.
1297fn html_img_span(el: &scraper::node::Element, link_href: Option<String>) -> Option<ParsedSpan> {
1298    let src = el.attr("src")?;
1299    if src.is_empty() {
1300        return None;
1301    }
1302    let dim = |name: &str| -> i64 {
1303        el.attr(name)
1304            .and_then(|v| v.trim().trim_end_matches("px").parse::<i64>().ok())
1305            .filter(|n| *n > 0)
1306            .unwrap_or(0)
1307    };
1308    Some(ParsedSpan {
1309        text: String::new(),
1310        link_href,
1311        image: Some(ParsedImage {
1312            src: src.to_string(),
1313            alt: el.attr("alt").unwrap_or_default().to_string(),
1314            width: dim("width"),
1315            height: dim("height"),
1316        }),
1317        ..Default::default()
1318    })
1319}
1320
1321/// Whether `node` is a `<ul>` or an `<ol>`.
1322fn is_html_list(node: &Node) -> bool {
1323    node.as_element()
1324        .is_some_and(|el| matches!(el.name(), "ul" | "ol"))
1325}
1326
1327/// Move every list item nested deeper than
1328/// [`MAX_LIST_LEVELS`](crate::parser_tools::list_depth::MAX_LIST_LEVELS)
1329/// lists into the deepest list kept, right after the item holding it, in
1330/// document order: a list nested past that depth comes out flattened to it,
1331/// every item kept, in its order.
1332///
1333/// The reader recurses a few elements deep per list level and stops at
1334/// `MAX_RECURSION_DEPTH`, so a list nested past some eighty levels lost every
1335/// item below that, pasted or loaded. The editing gestures nest no deeper than
1336/// this depth, and an insertion brings any list back to it (see `list_depth`),
1337/// so HTML holding more is read at that depth to begin with. The tree is
1338/// changed in place, without recursion, before it is read.
1339fn flatten_deep_html_lists(tree: &mut ego_tree::Tree<Node>) {
1340    use crate::parser_tools::list_depth::MAX_LIST_LEVELS;
1341    use ego_tree::iter::Edge;
1342
1343    // The lists one level past the deepest kept, in document order.
1344    let mut too_deep: Vec<ego_tree::NodeId> = Vec::new();
1345    let mut depth = 0usize;
1346    for edge in tree.root().traverse() {
1347        match edge {
1348            Edge::Open(node) if is_html_list(node.value()) => {
1349                if depth == MAX_LIST_LEVELS {
1350                    too_deep.push(node.id());
1351                }
1352                depth += 1;
1353            }
1354            Edge::Close(node) if is_html_list(node.value()) => {
1355                depth = depth.saturating_sub(1);
1356            }
1357            _ => {}
1358        }
1359    }
1360
1361    for list in too_deep {
1362        // The child of the deepest kept list that holds `list` (its `<li>`, or
1363        // `list` itself when a list sits right inside another): the items go
1364        // after it.
1365        let Some(list_ref) = tree.get(list) else {
1366            continue;
1367        };
1368        let mut holder = None;
1369        let mut current = list_ref;
1370        while let Some(parent) = current.parent() {
1371            if is_html_list(parent.value()) {
1372                holder = Some(current.id());
1373                break;
1374            }
1375            current = parent;
1376        }
1377        let Some(holder) = holder else {
1378            continue;
1379        };
1380        let mut after = holder;
1381        let items: Vec<ego_tree::NodeId> = list_ref
1382            .descendants()
1383            .filter(|node| {
1384                node.value()
1385                    .as_element()
1386                    .is_some_and(|el| el.name() == "li")
1387            })
1388            .map(|node| node.id())
1389            .collect();
1390        for item in items {
1391            // Each node placed here sits in the deepest kept list, so it has a
1392            // parent to be placed beside.
1393            if tree.get(after).and_then(|node| node.parent()).is_none() {
1394                break;
1395            }
1396            if let Some(mut node) = tree.get_mut(after) {
1397                node.insert_id_after(item);
1398                after = item;
1399            }
1400        }
1401        // What is left of the list is its items' wrappers, or text standing in
1402        // it outside any item. It goes after the items when it holds any text
1403        // (it is already beside them when it sits right in the deepest kept
1404        // list), and away when it holds none.
1405        let holds_text = tree.get(list).is_some_and(|node| {
1406            node.descendants()
1407                .any(|d| d.value().as_text().is_some_and(|t| !t.trim().is_empty()))
1408        });
1409        if !holds_text {
1410            if let Some(mut node) = tree.get_mut(list) {
1411                node.detach();
1412            }
1413        } else if holder != list
1414            && tree.get(after).and_then(|node| node.parent()).is_some()
1415            && let Some(mut node) = tree.get_mut(after)
1416        {
1417            node.insert_id_after(list);
1418        }
1419    }
1420}
1421
1422pub fn parse_html_elements(html: &str) -> Vec<ParsedElement> {
1423    use scraper::Html;
1424
1425    let mut fragment = Html::parse_fragment(html);
1426    flatten_deep_html_lists(&mut fragment.tree);
1427    let mut elements: Vec<ParsedElement> = Vec::new();
1428
1429    // Walk the DOM tree starting from the root
1430    let root = fragment.root_element();
1431
1432    #[derive(Clone, Default)]
1433    struct FmtState {
1434        bold: bool,
1435        italic: bool,
1436        underline: bool,
1437        strikeout: bool,
1438        code: bool,
1439        superscript: bool,
1440        subscript: bool,
1441        link_href: Option<String>,
1442    }
1443
1444    const MAX_RECURSION_DEPTH: usize = 256;
1445
1446    /// Elements whose text content is machinery, not prose.
1447    ///
1448    /// `<style>` and `<script>` hold raw text the HTML spec never renders, and
1449    /// `<head>` and its metadata children carry none a reader is meant to see.
1450    /// Every walker below recurses into unknown tags on the assumption that
1451    /// they are inline wrappers, which turns a stylesheet into paragraphs —
1452    /// and Word, Google Docs and Chrome all put a `<style>` block in the
1453    /// `text/html` flavour they publish to the clipboard, so this is the
1454    /// ordinary paste, not an exotic one.
1455    fn is_metadata_tag(tag: &str) -> bool {
1456        matches!(
1457            tag,
1458            "head"
1459                | "style"
1460                | "script"
1461                | "title"
1462                | "meta"
1463                | "link"
1464                | "base"
1465                | "noscript"
1466                | "template"
1467        )
1468    }
1469
1470    /// The text of everything under `node`, for a subtree nested deeper than
1471    /// the reader follows ([`MAX_RECURSION_DEPTH`]): read without recursing,
1472    /// with a space where a block ends so that words do not run together, and
1473    /// without what [`is_metadata_tag`] names.
1474    ///
1475    /// Past that depth the reader used to stop and keep nothing: text nested
1476    /// deeply enough in lists, quotations or plain `<div>`s was dropped from a
1477    /// paste without a trace.
1478    fn subtree_text(node: ego_tree::NodeRef<Node>) -> String {
1479        use ego_tree::iter::Edge;
1480        let mut text = String::new();
1481        // How many elements deep the walk is inside a metadata element.
1482        let mut hidden = 0usize;
1483        for edge in node.traverse() {
1484            match edge {
1485                Edge::Open(open) => match open.value() {
1486                    Node::Element(el) => {
1487                        if hidden > 0 || is_metadata_tag(el.name()) {
1488                            hidden += 1;
1489                        } else if !text.is_empty() && !text.ends_with(char::is_whitespace) {
1490                            text.push(' ');
1491                        }
1492                    }
1493                    Node::Text(t) if hidden == 0 => text.push_str(t),
1494                    _ => {}
1495                },
1496                Edge::Close(close) => {
1497                    if hidden > 0 && close.value().is_element() {
1498                        hidden -= 1;
1499                    }
1500                }
1501            }
1502        }
1503        text
1504    }
1505
1506    /// A span of `text` in `state`'s formatting.
1507    fn text_span(text: String, state: &FmtState) -> ParsedSpan {
1508        ParsedSpan {
1509            text,
1510            bold: state.bold,
1511            italic: state.italic,
1512            underline: state.underline,
1513            strikeout: state.strikeout,
1514            code: state.code,
1515            superscript: state.superscript,
1516            subscript: state.subscript,
1517            link_href: state.link_href.clone(),
1518            image: None,
1519            footnote_ref: None,
1520        }
1521    }
1522
1523    /// Collect inline spans from a `<td>` or `<th>` cell element.
1524    fn collect_cell_spans(
1525        node: ego_tree::NodeRef<Node>,
1526        state: &FmtState,
1527        spans: &mut Vec<ParsedSpan>,
1528        depth: usize,
1529    ) {
1530        if depth > MAX_RECURSION_DEPTH {
1531            let text = subtree_text(node);
1532            if !text.is_empty() {
1533                spans.push(text_span(text, state));
1534            }
1535            return;
1536        }
1537        for child in node.children() {
1538            match child.value() {
1539                Node::Text(text) => {
1540                    let t = text.text.to_string();
1541                    if !t.is_empty() {
1542                        spans.push(ParsedSpan {
1543                            text: t,
1544                            bold: state.bold,
1545                            italic: state.italic,
1546                            underline: state.underline,
1547                            strikeout: state.strikeout,
1548                            code: state.code,
1549                            superscript: state.superscript,
1550                            subscript: state.subscript,
1551                            link_href: state.link_href.clone(),
1552                            image: None,
1553                            footnote_ref: None,
1554                        });
1555                    }
1556                }
1557                Node::Element(el) => {
1558                    let tag = el.name();
1559                    if is_metadata_tag(tag) {
1560                        continue;
1561                    }
1562                    let mut new_state = state.clone();
1563                    match tag {
1564                        // Checked by attribute rather than by tag name: the
1565                        // producer picks the element (a `<sup>` renders sensibly
1566                        // in a browser), and only the attribute is a contract.
1567                        _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
1568                            if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
1569                            {
1570                                spans.push(span);
1571                            }
1572                            continue;
1573                        }
1574                        "b" | "strong" => new_state.bold = true,
1575                        "i" | "em" => new_state.italic = true,
1576                        "u" | "ins" => new_state.underline = true,
1577                        "s" | "del" | "strike" => new_state.strikeout = true,
1578                        "code" => new_state.code = true,
1579                        "sup" => new_state.superscript = true,
1580                        "sub" => new_state.subscript = true,
1581                        "a" => {
1582                            if let Some(href) = el.attr("href") {
1583                                new_state.link_href = Some(href.to_string());
1584                            }
1585                        }
1586                        "img" => {
1587                            if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
1588                                spans.push(span);
1589                            }
1590                            continue;
1591                        }
1592                        // A cell is read as one paragraph, so a line break, or a
1593                        // paragraph or list item of its own, is a space between
1594                        // words. Read as nothing, a cell of two paragraphs, which
1595                        // the HTML writer puts out as `One<br/>Two`, came back as
1596                        // the one word `OneTwo`.
1597                        "br" => {
1598                            spans.push(text_span(" ".to_string(), state));
1599                            continue;
1600                        }
1601                        "p" | "div" | "li" | "ul" | "ol" | "blockquote" | "pre" | "h1" | "h2"
1602                        | "h3" | "h4" | "h5" | "h6" => {
1603                            spans.push(text_span(" ".to_string(), state));
1604                        }
1605                        _ => {}
1606                    }
1607                    collect_cell_spans(child, &new_state, spans, depth + 1);
1608                }
1609                _ => {}
1610            }
1611        }
1612    }
1613
1614    /// The widest a `colspan` makes a row: HTML's own limit on the attribute. A row of
1615    /// ordinary cells may be wider; only the empty cells a span adds stop here.
1616    const MAX_HTML_COLSPAN: usize = 1000;
1617
1618    /// Parse a `<table>` element into a ParsedTable, with the number of `<td>` and `<th>`
1619    /// cells it has (see `push_table`).
1620    fn parse_table_element(table_node: ego_tree::NodeRef<Node>) -> (ParsedTable, usize) {
1621        let mut rows: Vec<Vec<ParsedTableCell>> = Vec::new();
1622        let mut header_rows: usize = 0;
1623        let mut own_cells: usize = 0;
1624
1625        fn collect_rows(
1626            node: ego_tree::NodeRef<Node>,
1627            rows: &mut Vec<Vec<ParsedTableCell>>,
1628            header_rows: &mut usize,
1629            own_cells: &mut usize,
1630            in_thead: bool,
1631        ) {
1632            for child in node.children() {
1633                if let Node::Element(el) = child.value() {
1634                    match el.name() {
1635                        "thead" => collect_rows(child, rows, header_rows, own_cells, true),
1636                        "tbody" | "tfoot" => {
1637                            collect_rows(child, rows, header_rows, own_cells, false)
1638                        }
1639                        "tr" => {
1640                            let mut cells: Vec<ParsedTableCell> = Vec::new();
1641                            // The cells of the row up to its last `<td>` or `<th>`: the
1642                            // empty cells a span adds after it hold no cell's place.
1643                            let mut own_end = 0;
1644                            for td in child.children() {
1645                                if let Node::Element(td_el) = td.value()
1646                                    && matches!(td_el.name(), "td" | "th")
1647                                {
1648                                    let mut spans = Vec::new();
1649                                    let state = FmtState::default();
1650                                    collect_cell_spans(td, &state, &mut spans, 0);
1651                                    collapse_inline_whitespace(&mut spans);
1652                                    if spans.is_empty() {
1653                                        spans.push(ParsedSpan::default());
1654                                    }
1655                                    cells.push(ParsedTableCell { spans });
1656                                    *own_cells += 1;
1657                                    own_end = cells.len();
1658                                    // A cell spanning columns covers the cells after it
1659                                    // in its row: empty ones hold its place, so the cells
1660                                    // that follow stay in their own columns and the
1661                                    // table is as wide as the columns it spans.
1662                                    let span = td_el
1663                                        .attr("colspan")
1664                                        .and_then(|value| value.trim().parse::<usize>().ok())
1665                                        .unwrap_or(1)
1666                                        .clamp(1, MAX_HTML_COLSPAN);
1667                                    for _ in 1..span {
1668                                        if cells.len() >= MAX_HTML_COLSPAN {
1669                                            break;
1670                                        }
1671                                        cells.push(ParsedTableCell {
1672                                            spans: vec![ParsedSpan::default()],
1673                                        });
1674                                    }
1675                                }
1676                            }
1677                            // A span closing a row widens the table only as far as another
1678                            // row's cells reach: `<th colspan="1000">` over rows of one
1679                            // cell is a table of one column, not of a thousand.
1680                            cells.truncate(own_end);
1681                            if !cells.is_empty() {
1682                                rows.push(cells);
1683                                if in_thead {
1684                                    *header_rows += 1;
1685                                }
1686                            }
1687                        }
1688                        _ => {}
1689                    }
1690                }
1691            }
1692        }
1693
1694        collect_rows(
1695            table_node,
1696            &mut rows,
1697            &mut header_rows,
1698            &mut own_cells,
1699            false,
1700        );
1701
1702        // Tables without explicit <thead> but with <th> cells: treat first row as header
1703        if header_rows == 0 && !rows.is_empty() {
1704            header_rows = 1;
1705        }
1706
1707        (
1708            ParsedTable {
1709                header_rows,
1710                rows,
1711                // The caller (`walk_node`) sets the real depth: this helper has
1712                // no visibility into the surrounding blockquote nesting.
1713                blockquote_depth: 0,
1714            },
1715            own_cells,
1716        )
1717    }
1718
1719    fn walk_node(
1720        node: ego_tree::NodeRef<Node>,
1721        state: &FmtState,
1722        elements: &mut Vec<ParsedElement>,
1723        current_list_style: &Option<ListStyle>,
1724        blockquote_depth: u32,
1725        list_depth: u32,
1726        depth: usize,
1727    ) {
1728        if depth > MAX_RECURSION_DEPTH {
1729            let text = subtree_text(node);
1730            if !text.trim().is_empty() {
1731                let mut spans = vec![text_span(text, state)];
1732                collapse_inline_whitespace(&mut spans);
1733                elements.push(ParsedElement::Block(ParsedBlock {
1734                    spans,
1735                    blockquote_depth,
1736                    ..ParsedBlock::default()
1737                }));
1738            }
1739            return;
1740        }
1741        match node.value() {
1742            Node::Element(el) => {
1743                let tag = el.name();
1744                if is_metadata_tag(tag) {
1745                    return;
1746                }
1747                let mut new_state = state.clone();
1748                let mut new_list_style = current_list_style.clone();
1749                let mut bq_depth = blockquote_depth;
1750                let mut new_list_depth = list_depth;
1751
1752                // Determine if this is a block-level element
1753                let is_block_tag = matches!(
1754                    tag,
1755                    "p" | "div"
1756                        | "h1"
1757                        | "h2"
1758                        | "h3"
1759                        | "h4"
1760                        | "h5"
1761                        | "h6"
1762                        | "li"
1763                        | "pre"
1764                        | "br"
1765                        | "blockquote"
1766                        | "body"
1767                        | "html"
1768                );
1769
1770                // Update formatting state
1771                match tag {
1772                    "b" | "strong" => new_state.bold = true,
1773                    "i" | "em" => new_state.italic = true,
1774                    "u" | "ins" => new_state.underline = true,
1775                    "s" | "del" | "strike" => new_state.strikeout = true,
1776                    "code" => new_state.code = true,
1777                    "sup" => new_state.superscript = true,
1778                    "sub" => new_state.subscript = true,
1779                    "a" => {
1780                        if let Some(href) = el.attr("href") {
1781                            new_state.link_href = Some(href.to_string());
1782                        }
1783                    }
1784                    "ul" => {
1785                        new_list_style = Some(ListStyle::Disc);
1786                        new_list_depth = list_depth + 1;
1787                    }
1788                    "ol" => {
1789                        new_list_style = Some(ListStyle::Decimal);
1790                        new_list_depth = list_depth + 1;
1791                    }
1792                    "blockquote" => {
1793                        bq_depth += 1;
1794                    }
1795                    _ => {}
1796                }
1797
1798                // Determine heading level
1799                let heading_level = match tag {
1800                    "h1" => Some(1),
1801                    "h2" => Some(2),
1802                    "h3" => Some(3),
1803                    "h4" => Some(4),
1804                    "h5" => Some(5),
1805                    "h6" => Some(6),
1806                    _ => None,
1807                };
1808
1809                let is_code_block = tag == "pre";
1810
1811                // Extract code language from <pre><code class="language-xxx">
1812                let code_language = if is_code_block {
1813                    node.children().find_map(|child| {
1814                        if let Node::Element(cel) = child.value()
1815                            && cel.name() == "code"
1816                            && let Some(cls) = cel.attr("class")
1817                        {
1818                            return cls
1819                                .split_whitespace()
1820                                .find_map(|c| c.strip_prefix("language-"))
1821                                .map(|l| l.to_string());
1822                        }
1823                        None
1824                    })
1825                } else {
1826                    None
1827                };
1828
1829                // Extract CSS styles from block-level elements
1830                let css = if is_block_tag {
1831                    el.attr("style").map(parse_block_styles).unwrap_or_default()
1832                } else {
1833                    BlockStyles::default()
1834                };
1835
1836                if tag == "table" {
1837                    // Parse table structure into a ParsedTable
1838                    let (mut parsed_table, own_cells) = parse_table_element(node);
1839                    if !parsed_table.rows.is_empty() {
1840                        parsed_table.blockquote_depth = bq_depth;
1841                        push_table(elements, parsed_table, own_cells);
1842                    }
1843                    return;
1844                }
1845
1846                if tag == "br" {
1847                    // <br> creates a new block
1848                    elements.push(ParsedElement::Block(ParsedBlock {
1849                        spans: vec![ParsedSpan {
1850                            text: String::new(),
1851                            ..Default::default()
1852                        }],
1853                        heading_level: None,
1854                        list_style: None,
1855                        list_indent: 0,
1856                        list_prefix: String::new(),
1857                        list_suffix: String::new(),
1858                        marker: None,
1859                        is_code_block: false,
1860                        code_language: None,
1861                        blockquote_depth: bq_depth,
1862                        line_height: None,
1863                        non_breakable_lines: None,
1864                        page_break_before: None,
1865                        direction: None,
1866                        background_color: None,
1867                        alignment: None,
1868                        top_margin: None,
1869                        text_indent: None,
1870                        semantic_role: None,
1871                    }));
1872                    return;
1873                }
1874
1875                if tag == "blockquote" {
1876                    // Blockquote is a container — recurse into children with increased depth
1877                    for child in node.children() {
1878                        walk_node(
1879                            child,
1880                            &new_state,
1881                            elements,
1882                            &new_list_style,
1883                            bq_depth,
1884                            new_list_depth,
1885                            depth + 1,
1886                        );
1887                    }
1888                } else if is_block_tag && tag != "br" {
1889                    // Start collecting spans for a new block.
1890                    // Use a temporary buffer so that nested block-level
1891                    // elements (e.g. sub-lists inside <li>) are collected
1892                    // separately and appended *after* the parent block.
1893                    let mut spans: Vec<ParsedSpan> = Vec::new();
1894                    let mut nested_elements: Vec<ParsedElement> = Vec::new();
1895                    collect_inline_spans(
1896                        node,
1897                        &new_state,
1898                        &mut spans,
1899                        &new_list_style,
1900                        &mut nested_elements,
1901                        bq_depth,
1902                        new_list_depth,
1903                        depth + 1,
1904                    );
1905
1906                    let list_style_for_block = if tag == "li" {
1907                        new_list_style.clone()
1908                    } else {
1909                        None
1910                    };
1911
1912                    let list_indent_for_block = if tag == "li" {
1913                        new_list_depth.saturating_sub(1)
1914                    } else {
1915                        0
1916                    };
1917
1918                    if !(is_code_block || css.preserve_whitespace == Some(true)) {
1919                        collapse_inline_whitespace(&mut spans);
1920                    }
1921
1922                    // Whitespace between block-level children is layout, not
1923                    // content. `<body>` holds a newline after every `</p>` an
1924                    // exporter writes, and keeping those as the container's own
1925                    // inline run is what closed a pasted book with a paragraph
1926                    // of four thousand blank lines. A block that produced no
1927                    // children keeps its empty run instead: `<p> </p>` is an
1928                    // empty paragraph the writer meant to be there.
1929                    let own_run_is_content =
1930                        spans_carry_content(&spans) || nested_elements.is_empty();
1931
1932                    if (!spans.is_empty() && own_run_is_content) || heading_level.is_some() {
1933                        // A code block is verbatim text, which no save can write an image or
1934                        // a note's reference into: a `<pre>` holding one is read as prose,
1935                        // which keeps it, one paragraph for each of its lines (see below).
1936                        let is_code_block = is_code_block
1937                            && !spans
1938                                .iter()
1939                                .any(|span| span.image.is_some() || span.footnote_ref.is_some());
1940                        let block = ParsedBlock {
1941                            spans,
1942                            heading_level,
1943                            list_style: list_style_for_block,
1944                            list_indent: list_indent_for_block,
1945                            list_prefix: String::new(),
1946                            list_suffix: String::new(),
1947                            marker: None,
1948                            is_code_block,
1949                            code_language: code_language.filter(|_| is_code_block),
1950                            blockquote_depth: bq_depth,
1951                            line_height: css.line_height,
1952                            non_breakable_lines: css.non_breakable_lines,
1953                            page_break_before: css.page_break_before,
1954                            direction: css.direction,
1955                            background_color: css.background_color,
1956                            alignment: None,
1957                            top_margin: None,
1958                            text_indent: None,
1959                            semantic_role: None,
1960                        };
1961                        // Kept whitespace keeps its line breaks, and a line break in
1962                        // prose is a new block to the model. Only a code block holds
1963                        // its lines in one block.
1964                        if is_code_block {
1965                            elements.push(ParsedElement::Block(block));
1966                        } else {
1967                            elements.extend(
1968                                split_block_at_line_breaks(block)
1969                                    .into_iter()
1970                                    .map(ParsedElement::Block),
1971                            );
1972                        }
1973                    }
1974                    // Append nested block elements after the parent block
1975                    elements.append(&mut nested_elements);
1976                } else if matches!(tag, "ul" | "ol" | "thead" | "tbody" | "tr") {
1977                    // Container elements: recurse into children
1978                    for child in node.children() {
1979                        walk_node(
1980                            child,
1981                            &new_state,
1982                            elements,
1983                            &new_list_style,
1984                            bq_depth,
1985                            new_list_depth,
1986                            depth + 1,
1987                        );
1988                    }
1989                } else {
1990                    // Inline element or unknown: recurse
1991                    for child in node.children() {
1992                        walk_node(
1993                            child,
1994                            &new_state,
1995                            elements,
1996                            current_list_style,
1997                            bq_depth,
1998                            list_depth,
1999                            depth + 1,
2000                        );
2001                    }
2002                }
2003            }
2004            Node::Text(text) => {
2005                let t = text.text.to_string();
2006                let trimmed = t.trim();
2007                if !trimmed.is_empty() {
2008                    // Bare text not in a block — create a paragraph
2009                    elements.push(ParsedElement::Block(ParsedBlock {
2010                        spans: vec![ParsedSpan {
2011                            text: trimmed.to_string(),
2012                            bold: state.bold,
2013                            italic: state.italic,
2014                            underline: state.underline,
2015                            strikeout: state.strikeout,
2016                            code: state.code,
2017                            superscript: state.superscript,
2018                            subscript: state.subscript,
2019                            link_href: state.link_href.clone(),
2020                            image: None,
2021                            footnote_ref: None,
2022                        }],
2023                        heading_level: None,
2024                        list_style: None,
2025                        list_indent: 0,
2026                        list_prefix: String::new(),
2027                        list_suffix: String::new(),
2028                        marker: None,
2029                        is_code_block: false,
2030                        code_language: None,
2031                        blockquote_depth,
2032                        line_height: None,
2033                        non_breakable_lines: None,
2034                        page_break_before: None,
2035                        direction: None,
2036                        background_color: None,
2037                        alignment: None,
2038                        top_margin: None,
2039                        text_indent: None,
2040                        semantic_role: None,
2041                    }));
2042                }
2043            }
2044            _ => {
2045                // Document, Comment, etc. — recurse children
2046                for child in node.children() {
2047                    walk_node(
2048                        child,
2049                        state,
2050                        elements,
2051                        current_list_style,
2052                        blockquote_depth,
2053                        list_depth,
2054                        depth + 1,
2055                    );
2056                }
2057            }
2058        }
2059    }
2060
2061    /// Collect inline spans from a block-level element's children.
2062    /// If a nested block-level element is encountered, it is flushed as a
2063    /// separate block.
2064    #[allow(clippy::too_many_arguments)]
2065    fn collect_inline_spans(
2066        node: ego_tree::NodeRef<Node>,
2067        state: &FmtState,
2068        spans: &mut Vec<ParsedSpan>,
2069        current_list_style: &Option<ListStyle>,
2070        elements: &mut Vec<ParsedElement>,
2071        blockquote_depth: u32,
2072        list_depth: u32,
2073        depth: usize,
2074    ) {
2075        if depth > MAX_RECURSION_DEPTH {
2076            let text = subtree_text(node);
2077            if !text.is_empty() {
2078                spans.push(text_span(text, state));
2079            }
2080            return;
2081        }
2082        for child in node.children() {
2083            match child.value() {
2084                Node::Text(text) => {
2085                    let t = text.text.to_string();
2086                    if !t.is_empty() {
2087                        spans.push(ParsedSpan {
2088                            text: t,
2089                            bold: state.bold,
2090                            italic: state.italic,
2091                            underline: state.underline,
2092                            strikeout: state.strikeout,
2093                            code: state.code,
2094                            superscript: state.superscript,
2095                            subscript: state.subscript,
2096                            link_href: state.link_href.clone(),
2097                            image: None,
2098                            footnote_ref: None,
2099                        });
2100                    }
2101                }
2102                Node::Element(el) => {
2103                    let tag = el.name();
2104                    if is_metadata_tag(tag) {
2105                        continue;
2106                    }
2107                    let mut new_state = state.clone();
2108
2109                    match tag {
2110                        // Checked by attribute rather than by tag name: the
2111                        // producer picks the element (a `<sup>` renders sensibly
2112                        // in a browser), and only the attribute is a contract.
2113                        _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
2114                            if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
2115                            {
2116                                spans.push(span);
2117                            }
2118                            continue;
2119                        }
2120                        "b" | "strong" => new_state.bold = true,
2121                        "i" | "em" => new_state.italic = true,
2122                        "u" | "ins" => new_state.underline = true,
2123                        "s" | "del" | "strike" => new_state.strikeout = true,
2124                        "code" => new_state.code = true,
2125                        "sup" => new_state.superscript = true,
2126                        "sub" => new_state.subscript = true,
2127                        "a" => {
2128                            if let Some(href) = el.attr("href") {
2129                                new_state.link_href = Some(href.to_string());
2130                            }
2131                        }
2132                        "img" => {
2133                            if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
2134                                spans.push(span);
2135                            }
2136                            continue;
2137                        }
2138                        _ => {}
2139                    }
2140
2141                    // Check for nested block elements
2142                    let nested_block = matches!(
2143                        tag,
2144                        "p" | "div"
2145                            | "h1"
2146                            | "h2"
2147                            | "h3"
2148                            | "h4"
2149                            | "h5"
2150                            | "h6"
2151                            | "li"
2152                            | "pre"
2153                            | "blockquote"
2154                            | "ul"
2155                            | "ol"
2156                    );
2157
2158                    if tag == "br" {
2159                        // br within a block: treat as splitting into new block
2160                        // For simplicity, just add a newline to current span
2161                        spans.push(ParsedSpan {
2162                            text: String::new(),
2163                            ..Default::default()
2164                        });
2165                    } else if nested_block || tag == "table" {
2166                        // Flush as separate element
2167                        walk_node(
2168                            child,
2169                            &new_state,
2170                            elements,
2171                            current_list_style,
2172                            blockquote_depth,
2173                            list_depth,
2174                            depth + 1,
2175                        );
2176                    } else {
2177                        // Inline element: recurse
2178                        collect_inline_spans(
2179                            child,
2180                            &new_state,
2181                            spans,
2182                            current_list_style,
2183                            elements,
2184                            blockquote_depth,
2185                            list_depth,
2186                            depth + 1,
2187                        );
2188                    }
2189                }
2190                _ => {}
2191            }
2192        }
2193    }
2194
2195    let initial_state = FmtState::default();
2196    // Treat the root element as a block-level container so that
2197    // top-level inline elements (e.g. `<b>Bold</b> <em>Italic</em>`)
2198    // are grouped into a single block instead of becoming separate blocks.
2199    let mut root_spans: Vec<ParsedSpan> = Vec::new();
2200    collect_inline_spans(
2201        *root,
2202        &initial_state,
2203        &mut root_spans,
2204        &None,
2205        &mut elements,
2206        0,
2207        0,
2208        0,
2209    );
2210    collapse_inline_whitespace(&mut root_spans);
2211    // Only when something survived: the root's direct text children are the
2212    // newlines between top-level blocks in every document an exporter writes.
2213    if spans_carry_content(&root_spans) {
2214        elements.push(ParsedElement::Block(ParsedBlock {
2215            spans: root_spans,
2216            heading_level: None,
2217            list_style: None,
2218            list_indent: 0,
2219            list_prefix: String::new(),
2220            list_suffix: String::new(),
2221            marker: None,
2222            is_code_block: false,
2223            code_language: None,
2224            blockquote_depth: 0,
2225            line_height: None,
2226            non_breakable_lines: None,
2227            page_break_before: None,
2228            direction: None,
2229            background_color: None,
2230            alignment: None,
2231            top_margin: None,
2232            text_indent: None,
2233            semantic_role: None,
2234        }));
2235    }
2236
2237    // If no elements were parsed, create a single empty paragraph
2238    if elements.is_empty() {
2239        elements.push(ParsedElement::Block(ParsedBlock {
2240            spans: vec![ParsedSpan {
2241                text: String::new(),
2242                ..Default::default()
2243            }],
2244            heading_level: None,
2245            list_style: None,
2246            list_indent: 0,
2247            list_prefix: String::new(),
2248            list_suffix: String::new(),
2249            marker: None,
2250            is_code_block: false,
2251            code_language: None,
2252            blockquote_depth: 0,
2253            line_height: None,
2254            non_breakable_lines: None,
2255            page_break_before: None,
2256            direction: None,
2257            background_color: None,
2258            alignment: None,
2259            top_margin: None,
2260            text_indent: None,
2261            semantic_role: None,
2262        }));
2263    }
2264
2265    elements
2266}
2267
2268/// Convert a `ParsedSpan` (parser output) into the `CharacterFormat` used by
2269/// `FormatRun`. `is_code_block` forces `monospace` as the font family for
2270/// every span inside a code block.
2271pub fn character_format_from_span(
2272    span: &ParsedSpan,
2273    is_code_block: bool,
2274) -> crate::format_runs::CharacterFormat {
2275    use crate::entities::CharVerticalAlignment;
2276    crate::format_runs::CharacterFormat {
2277        font_bold: if span.bold { Some(true) } else { None },
2278        font_italic: if span.italic { Some(true) } else { None },
2279        font_underline: if span.underline { Some(true) } else { None },
2280        font_strikeout: if span.strikeout { Some(true) } else { None },
2281        font_family: if span.code || is_code_block {
2282            Some("monospace".to_string())
2283        } else {
2284            None
2285        },
2286        anchor_href: span.link_href.clone(),
2287        is_anchor: if span.link_href.is_some() {
2288            Some(true)
2289        } else {
2290            None
2291        },
2292        vertical_alignment: if span.superscript {
2293            Some(CharVerticalAlignment::SuperScript)
2294        } else if span.subscript {
2295            Some(CharVerticalAlignment::SubScript)
2296        } else {
2297            None
2298        },
2299        ..Default::default()
2300    }
2301}
2302
2303/// Translate a slice of parsed spans into `(plain_text, format_runs)`.
2304///
2305/// One non-default span yields one `FormatRun`; spans with empty
2306/// `CharacterFormat` (no decoration, no link, no code) emit no run, since an
2307/// absent run means "inherit default formatting" in the new model. Adjacent
2308/// runs with identical formats are coalesced via `coalesce_in_place` so the
2309/// resulting vector satisfies `debug_assert_well_formed`.
2310///
2311/// Returns the concatenated `plain_text` of all spans and a sorted,
2312/// non-overlapping, coalesced `Vec<FormatRun>`. Both safe to feed straight
2313/// into the store under the dual-write bridge.
2314pub fn format_runs_from_spans(spans: &[ParsedSpan], is_code_block: bool) -> ParsedInline {
2315    use crate::format_runs::{
2316        CharacterFormat, FootnoteRefAnchor, FormatRun, ImageAnchor, coalesce_in_place,
2317    };
2318
2319    let mut plain_text = String::new();
2320    let mut runs: Vec<FormatRun> = Vec::new();
2321    let mut images: Vec<ImageAnchor> = Vec::new();
2322    let mut footnote_refs: Vec<FootnoteRefAnchor> = Vec::new();
2323    let default = CharacterFormat::default();
2324
2325    for span in spans {
2326        let byte_start = plain_text.len() as u32;
2327
2328        if let Some(label) = &span.footnote_ref {
2329            // A reference occupies one U+FFFC, exactly as an image does, so
2330            // every downstream offset treats the two alike.
2331            plain_text.push('\u{FFFC}');
2332            // Raised, always — whatever the surrounding run is doing.
2333            //
2334            // A footnote marker is superscript by definition, in every
2335            // typographic tradition and in every reader that renders djot. The
2336            // ambient `superscript` flag the span carries is the *prose's*, and
2337            // prose is not superscript, so taking it verbatim sets a note's
2338            // number on the baseline in the middle of a sentence — which reads
2339            // as a stray digit the writer typed rather than as a reference.
2340            //
2341            // It is set here, on the anchor, rather than at render time so that
2342            // every consumer agrees: the editor raises it, the exporters that
2343            // carry character formatting carry it, and the djot writer knows to
2344            // emit `[^label]` *without* wrapping it in `^…^` (see
2345            // `a_reference_is_not_wrapped_in_superscript_markers`).
2346            let mut format = character_format_from_span(span, is_code_block);
2347            format.vertical_alignment = Some(crate::entities::CharVerticalAlignment::SuperScript);
2348            footnote_refs.push(FootnoteRefAnchor {
2349                byte_offset: byte_start,
2350                label: label.clone(),
2351                format,
2352            });
2353            continue;
2354        }
2355
2356        if let Some(image) = &span.image {
2357            // An image occupies one U+FFFC in the text, exactly as
2358            // `insert_image` mirrors into the rope, so every downstream offset
2359            // calculation treats a parsed image and an inserted one alike.
2360            plain_text.push('\u{FFFC}');
2361            images.push(ImageAnchor {
2362                byte_offset: byte_start,
2363                name: image.src.clone(),
2364                alt: image.alt.clone(),
2365                width: image.width,
2366                height: image.height,
2367                quality: 100,
2368                format: character_format_from_span(span, is_code_block),
2369            });
2370            continue;
2371        }
2372
2373        plain_text.push_str(&span.text);
2374        let byte_end = plain_text.len() as u32;
2375        if byte_start == byte_end {
2376            continue;
2377        }
2378        let format = character_format_from_span(span, is_code_block);
2379        if format == default {
2380            continue;
2381        }
2382        runs.push(FormatRun {
2383            byte_start,
2384            byte_end,
2385            format,
2386        });
2387    }
2388    coalesce_in_place(&mut runs);
2389    ParsedInline {
2390        plain_text,
2391        runs,
2392        images,
2393        footnote_refs,
2394    }
2395}
2396
2397/// The three parallel things a block stores, as recovered from parsed spans.
2398///
2399/// Returned as a struct rather than a tuple because it grew a third member
2400/// (images) after nine call sites already destructured a pair — and every one
2401/// of those sites has to decide what to do with images, so a silent
2402/// tuple-arity change would have been the wrong kind of easy.
2403#[derive(Debug, Clone, Default)]
2404pub struct ParsedInline {
2405    pub plain_text: String,
2406    pub runs: Vec<crate::format_runs::FormatRun>,
2407    pub images: Vec<crate::format_runs::ImageAnchor>,
2408    pub footnote_refs: Vec<crate::format_runs::FootnoteRefAnchor>,
2409}
2410
2411// ─── Djot parsing ────────────────────────────────────────────────────
2412
2413/// Map a jotdown unordered/task bullet marker to a model `ListStyle`.
2414///
2415/// The mapping is a stable bijection (`-`↔Disc, `*`↔Circle, `+`↔Square) so the
2416/// djot exporter can recover the exact bullet character for a lossless
2417/// round-trip.
2418fn djot_bullet_style(b: jotdown::ListBulletType) -> ListStyle {
2419    use jotdown::ListBulletType as B;
2420    match b {
2421        B::Dash => ListStyle::Disc,
2422        B::Star => ListStyle::Circle,
2423        B::Plus => ListStyle::Square,
2424    }
2425}
2426
2427/// Map a jotdown ordered-list numbering scheme to a model `ListStyle`.
2428fn djot_ordered_style(n: jotdown::OrderedListNumbering) -> ListStyle {
2429    use jotdown::OrderedListNumbering as N;
2430    match n {
2431        N::Decimal => ListStyle::Decimal,
2432        N::AlphaLower => ListStyle::LowerAlpha,
2433        N::AlphaUpper => ListStyle::UpperAlpha,
2434        N::RomanLower => ListStyle::LowerRoman,
2435        N::RomanUpper => ListStyle::UpperRoman,
2436    }
2437}
2438
2439/// Map a jotdown ordered-list delimiter to the `(prefix, suffix)` affixes
2440/// stored on the `List` entity (`1.` → `("", ".")`, `1)` → `("", ")")`,
2441/// `(1)` → `("(", ")")`).
2442fn djot_ordered_affixes(style: jotdown::OrderedListStyle) -> (String, String) {
2443    use jotdown::OrderedListStyle as S;
2444    match style {
2445        S::Period => (String::new(), ".".to_string()),
2446        S::Paren => (String::new(), ")".to_string()),
2447        S::ParenParen => ("(".to_string(), ")".to_string()),
2448    }
2449}
2450
2451/// Optional block-level style attributes carried on a djot block through its
2452/// `{key=value}` block attributes. All `None` when the block has no such
2453/// attributes (or they were filtered out by [`DjotImportOptions`]).
2454#[derive(Debug, Clone, Default)]
2455struct DjotBlockStyle {
2456    alignment: Option<Alignment>,
2457    line_height: Option<i64>,
2458    non_breakable_lines: Option<bool>,
2459    page_break_before: Option<bool>,
2460    direction: Option<TextDirection>,
2461    background_color: Option<String>,
2462    top_margin: Option<i64>,
2463    text_indent: Option<i64>,
2464    semantic_role: Option<SemanticRole>,
2465}
2466
2467impl DjotBlockStyle {
2468    /// Overlay the `Some` fields of `other` onto `self`, leaving `self`'s
2469    /// existing values for any field `other` does not set. Used to combine a
2470    /// heading's enclosing-`Section` attributes with any on the heading itself.
2471    fn merge_from(&mut self, other: DjotBlockStyle) {
2472        if other.alignment.is_some() {
2473            self.alignment = other.alignment;
2474        }
2475        if other.line_height.is_some() {
2476            self.line_height = other.line_height;
2477        }
2478        if other.non_breakable_lines.is_some() {
2479            self.non_breakable_lines = other.non_breakable_lines;
2480        }
2481        if other.page_break_before.is_some() {
2482            self.page_break_before = other.page_break_before;
2483        }
2484        if other.direction.is_some() {
2485            self.direction = other.direction;
2486        }
2487        if other.background_color.is_some() {
2488            self.background_color = other.background_color;
2489        }
2490        if other.top_margin.is_some() {
2491            self.top_margin = other.top_margin;
2492        }
2493        if other.text_indent.is_some() {
2494            self.text_indent = other.text_indent;
2495        }
2496        if other.semantic_role.is_some() {
2497            self.semantic_role = other.semantic_role.clone();
2498        }
2499    }
2500}
2501
2502/// Read the round-tripped block-style attributes off a djot block's
2503/// [`jotdown::Attributes`], honouring the import [`DjotImportOptions`]. Keys are
2504/// the model field names (`alignment`, `line_height`, `direction`,
2505/// `non_breakable_lines`, `page_break_before`, `background_color`, `top_margin`,
2506/// `text_indent`, `semantic_role`); unrecognised values are ignored.
2507fn block_attrs_to_style(attrs: &jotdown::Attributes, opts: &DjotImportOptions) -> DjotBlockStyle {
2508    let mut style = DjotBlockStyle::default();
2509
2510    if opts.alignment
2511        && let Some(v) = attrs.get_value("alignment")
2512    {
2513        style.alignment = match v.to_string().as_str() {
2514            "left" => Some(Alignment::Left),
2515            "right" => Some(Alignment::Right),
2516            "center" => Some(Alignment::Center),
2517            "justify" => Some(Alignment::Justify),
2518            _ => None,
2519        };
2520    }
2521    if opts.line_height
2522        && let Some(v) = attrs.get_value("line_height")
2523    {
2524        style.line_height = v.to_string().parse::<i64>().ok();
2525    }
2526    if opts.direction
2527        && let Some(v) = attrs.get_value("direction")
2528    {
2529        style.direction = match v.to_string().as_str() {
2530            "ltr" => Some(TextDirection::LeftToRight),
2531            "rtl" => Some(TextDirection::RightToLeft),
2532            _ => None,
2533        };
2534    }
2535    if opts.non_breakable_lines
2536        && let Some(v) = attrs.get_value("non_breakable_lines")
2537    {
2538        style.non_breakable_lines = match v.to_string().as_str() {
2539            "true" => Some(true),
2540            "false" => Some(false),
2541            _ => None,
2542        };
2543    }
2544    if opts.page_break_before
2545        && let Some(v) = attrs.get_value("page_break_before")
2546    {
2547        style.page_break_before = match v.to_string().as_str() {
2548            "true" => Some(true),
2549            "false" => Some(false),
2550            _ => None,
2551        };
2552    }
2553    if opts.background_color
2554        && let Some(v) = attrs.get_value("background_color")
2555    {
2556        style.background_color = Some(v.to_string());
2557    }
2558    if opts.top_margin
2559        && let Some(v) = attrs.get_value("top_margin")
2560    {
2561        style.top_margin = v.to_string().parse::<i64>().ok();
2562    }
2563    if opts.text_indent
2564        && let Some(v) = attrs.get_value("text_indent")
2565    {
2566        style.text_indent = v.to_string().parse::<i64>().ok();
2567    }
2568    if opts.semantic_role
2569        && let Some(v) = attrs.get_value("semantic_role")
2570    {
2571        style.semantic_role = match v.to_string().as_str() {
2572            "epigraph" => Some(SemanticRole::Epigraph),
2573            // An unknown role is dropped, not guessed at — the same way an unknown
2574            // alignment value above is. A future role read by an older build then
2575            // degrades to a plain blockquote, which is what it looks like anyway.
2576            _ => None,
2577        };
2578    }
2579
2580    style
2581}
2582
2583/// Push a finished block into `elements`, applying the djot block-level fields
2584/// plus any round-tripped block-style attributes carried in `style`.
2585#[allow(clippy::too_many_arguments)]
2586fn djot_push_block(
2587    elements: &mut Vec<ParsedElement>,
2588    spans: Vec<ParsedSpan>,
2589    heading_level: Option<i64>,
2590    list_style: Option<ListStyle>,
2591    list_indent: u32,
2592    list_prefix: String,
2593    list_suffix: String,
2594    marker: Option<MarkerType>,
2595    is_code_block: bool,
2596    code_language: Option<String>,
2597    blockquote_depth: u32,
2598    style: DjotBlockStyle,
2599) {
2600    elements.push(ParsedElement::Block(ParsedBlock {
2601        spans,
2602        heading_level,
2603        list_style,
2604        list_indent,
2605        list_prefix,
2606        list_suffix,
2607        marker,
2608        is_code_block,
2609        code_language,
2610        blockquote_depth,
2611        line_height: style.line_height,
2612        non_breakable_lines: style.non_breakable_lines,
2613        page_break_before: style.page_break_before,
2614        direction: style.direction,
2615        background_color: style.background_color,
2616        alignment: style.alignment,
2617        top_margin: style.top_margin,
2618        text_indent: style.text_indent,
2619        semantic_role: style.semantic_role.clone(),
2620    }));
2621}
2622
2623/// A Djot document whose structure cannot be computed, as its source: one plain
2624/// paragraph for each line that holds text, with that line's text as it is written, its
2625/// leading and trailing whitespace aside.
2626///
2627/// A paragraph a line is what keeps the text the same through a save. The Djot writer
2628/// escapes whatever a paragraph opens with and whatever in it would read as markup, so
2629/// the saved text reads back as these same paragraphs. Shown as one paragraph, the
2630/// source kept its line breaks through a save, read back as the same document the parser
2631/// could not take, and every save escaped its backslashes once more. Blank lines are left
2632/// out because the reader keeps no block for an empty paragraph, and a line's outer
2633/// whitespace because it is the source's indentation, which the parser does not read as
2634/// text in a line it can take either.
2635fn djot_source_lines(djot: &str) -> Vec<ParsedElement> {
2636    djot.lines()
2637        .map(|line| line.trim_matches(|c: char| c.is_ascii_whitespace()))
2638        .filter(|line| !line.is_empty())
2639        .map(|line| {
2640            ParsedElement::Block(ParsedBlock {
2641                spans: vec![ParsedSpan {
2642                    text: line.to_string(),
2643                    ..Default::default()
2644                }],
2645                ..Default::default()
2646            })
2647        })
2648        .collect()
2649}
2650
2651/// Parse djot source into the shared [`ParsedElement`] intermediate, mirroring
2652/// [`parse_markdown`]. Uses the [`jotdown`] pull parser.
2653///
2654/// Constructs the document model cannot represent are dropped, and their text
2655/// content is discarded so it never leaks into the document: footnotes, math,
2656/// fenced divs, raw blocks/inline, thematic breaks, description lists,
2657/// captions, symbols, link-reference definitions, and highlight/`mark`. Inline
2658/// images keep their alt text as plain text (the image itself is not modelled),
2659/// matching the Markdown importer. Smart-punctuation events are normalised to
2660/// their canonical Unicode characters so the model→djot→model round-trip is a
2661/// fixpoint.
2662///
2663/// Standalone paragraphs and headings additionally carry the optional
2664/// block-style attributes selected by `options` — paragraph alignment, line
2665/// height, text direction, non-breakable lines and background color — read from
2666/// djot `{key=value}` block attributes (see [`DjotImportOptions`]). List items,
2667/// code blocks and table cells normalise their block styling away.
2668///
2669/// Known model limitations (normalised, not preserved on round-trip):
2670/// ordered-list start number, table column alignment, and list tight/loose.
2671pub fn parse_djot(djot: &str, options: &DjotImportOptions) -> Vec<ParsedElement> {
2672    use jotdown::{Container as C, Event as E, ListKind};
2673
2674    // `jotdown` recurses with no limit of its own, once per nested container and
2675    // once per line of a paragraph that leaves an inline opener waiting, and a
2676    // stack overflow **aborts the process**: it cannot be caught by
2677    // `catch_unwind`, so there is no recovering from it after the fact and the
2678    // only safe move is not to start. It also panics on a heading deeper than it
2679    // can number, which a caller on an interface thread does not survive either.
2680    // Input is not always the author's own: a project bundle is mailed and
2681    // shared, and an imported document comes from whoever sent it.
2682    //
2683    // `djot_depth::parsable` is the door that keeps the parser within its limits,
2684    // and it degrades rather than refuses, so the signature stays the same and no
2685    // word is lost. Such a heading is escaped into text, as it was written. A
2686    // document nested too deeply by its indentation, which is how a deeply nested
2687    // list is written, is read with the indentation cut: the list comes out
2688    // flattened below a fixed depth, every item and word kept. This used to hand
2689    // back a pasted list nested two hundred levels as one paragraph of markup. A
2690    // long paragraph is parsed on a stack sized for it, and the block holding one
2691    // too long for any stack is shown as its lines, the rest of the document as it
2692    // was.
2693    //
2694    // A document still nested too deeply once flattened, or one the parser panicked
2695    // on, comes back as its lines, one plain paragraph each (see
2696    // `djot_source_lines`). Its structure is not computed, which is the honest
2697    // answer for a document whose structure cannot be computed without ending the
2698    // process, and its text is still there for the writer to see and repair.
2699    let parsable = crate::parser_tools::djot_depth::parsable(djot);
2700    let Some(events) = parsable.as_ref().and_then(|parsable| parsable.events()) else {
2701        return djot_source_lines(djot);
2702    };
2703
2704    let mut elements: Vec<ParsedElement> = Vec::new();
2705    let mut current_spans: Vec<ParsedSpan> = Vec::new();
2706    let mut current_heading: Option<i64> = None;
2707    let mut is_code_block = false;
2708    let mut code_language: Option<String> = None;
2709    let mut blockquote_depth: u32 = 0;
2710    // Block-style attributes captured from a standalone paragraph/heading's djot
2711    // `{…}` block attributes, consumed when that block is flushed.
2712    let mut pending_style = DjotBlockStyle::default();
2713
2714    // Inline formatting state.
2715    let mut bold = false;
2716    let mut italic = false;
2717    let mut underline = false;
2718    let mut strikeout = false;
2719    let mut code = false;
2720    let mut superscript = false;
2721    let mut subscript = false;
2722    let mut link_href: Option<String> = None;
2723    // Set between an image's Start and End. Its alt text arrives as ordinary
2724    // `Str` events in between, so it has to be captured rather than emitted.
2725    let mut pending_image: Option<ParsedImage> = None;
2726
2727    // List nesting: each entry is (style, prefix, suffix); depth = indent + 1.
2728    let mut list_stack: Vec<(ListStyle, String, String)> = Vec::new();
2729    // Context applied to the next flushed block while inside a list item.
2730    let mut cur_list_style: Option<ListStyle> = None;
2731    let mut cur_list_prefix = String::new();
2732    let mut cur_list_suffix = String::new();
2733    let mut cur_list_indent: u32 = 0;
2734    let mut cur_marker: Option<MarkerType> = None;
2735
2736    // Table accumulation.
2737    let mut in_table_cell = false;
2738    let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
2739    let mut current_row: Vec<ParsedTableCell> = Vec::new();
2740    let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
2741    let mut table_header_rows: usize = 0;
2742    let mut row_is_head = false;
2743
2744    // Subtree-skip depth for unrepresentable containers (their entire content
2745    // is dropped). Incremented on the dropped container's `Start` and on every
2746    // nested `Start`; decremented on every `End`.
2747    let mut skip_depth: u32 = 0;
2748
2749    // The label and element index of the footnote definition currently open,
2750    // if any. Djot has no footnote inside a footnote, so one slot is enough
2751    // where the dropped containers need a depth counter.
2752    let mut footnote_open: Option<(String, usize)> = None;
2753
2754    // Push one inline span carrying the current formatting state into the
2755    // active sink (table cell or block). A macro (not a closure) to avoid
2756    // borrowing `current_spans`/`current_cell_spans` across the formatting
2757    // state reads.
2758    macro_rules! push_text {
2759        ($t:expr) => {{
2760            // Alt text belongs to the image, not to the paragraph. While an
2761            // image is open every text event is diverted into its description,
2762            // which is what keeps a photo's caption out of the manuscript's
2763            // word count and out of the search corpus.
2764            if let Some(img) = pending_image.as_mut() {
2765                img.alt.push_str(($t).as_ref());
2766            } else {
2767                let sp = ParsedSpan {
2768                    text: ($t).to_string(),
2769                    bold,
2770                    italic,
2771                    underline,
2772                    strikeout,
2773                    code,
2774                    superscript,
2775                    subscript,
2776                    link_href: link_href.clone(),
2777                    image: None,
2778                    footnote_ref: None,
2779                };
2780                if in_table_cell {
2781                    current_cell_spans.push(sp);
2782                } else {
2783                    current_spans.push(sp);
2784                }
2785            }
2786        }};
2787    }
2788
2789    // Push a completed inline image span into the active sink.
2790    macro_rules! push_image {
2791        ($img:expr) => {{
2792            let sp = ParsedSpan {
2793                text: String::new(),
2794                bold,
2795                italic,
2796                underline,
2797                strikeout,
2798                code,
2799                superscript,
2800                subscript,
2801                link_href: link_href.clone(),
2802                image: Some($img),
2803                footnote_ref: None,
2804            };
2805            if in_table_cell {
2806                current_cell_spans.push(sp);
2807            } else {
2808                current_spans.push(sp);
2809            }
2810        }};
2811    }
2812
2813    // Enter a list item, flushing any unterminated inline content first and
2814    // capturing the list context + task marker for the item's block.
2815    macro_rules! enter_item {
2816        ($marker:expr) => {{
2817            if !current_spans.is_empty() {
2818                djot_push_block(
2819                    &mut elements,
2820                    std::mem::take(&mut current_spans),
2821                    None,
2822                    cur_list_style.clone(),
2823                    cur_list_indent,
2824                    cur_list_prefix.clone(),
2825                    cur_list_suffix.clone(),
2826                    cur_marker.clone(),
2827                    false,
2828                    None,
2829                    blockquote_depth,
2830                    DjotBlockStyle::default(),
2831                );
2832            }
2833            let (style, prefix, suffix) = list_stack.last().cloned().unwrap_or((
2834                ListStyle::Disc,
2835                String::new(),
2836                String::new(),
2837            ));
2838            cur_list_style = Some(style);
2839            cur_list_prefix = prefix;
2840            cur_list_suffix = suffix;
2841            cur_list_indent = list_stack.len().saturating_sub(1) as u32;
2842            cur_marker = $marker;
2843        }};
2844    }
2845
2846    // A line break inside a block: a new block, which is what the model means by one,
2847    // with the list, quotation, heading and block style the line was in. A table cell
2848    // is one line, so there it is a space.
2849    //
2850    // The page break is the one style that stays with the first line: it is where the
2851    // block starts, and copied to every line it put each on a page of its own.
2852    macro_rules! line_break {
2853        () => {{
2854            if in_table_cell {
2855                push_text!(" ");
2856            } else if !current_spans.is_empty() && current_heading.is_some() {
2857                // The line before a hard break in a heading is a line of that
2858                // heading, as the line after it is when the heading ends. Read as
2859                // a paragraph, a heading written over two lines came back as a
2860                // paragraph and a heading.
2861                djot_push_block(
2862                    &mut elements,
2863                    std::mem::take(&mut current_spans),
2864                    current_heading,
2865                    None,
2866                    0,
2867                    String::new(),
2868                    String::new(),
2869                    None,
2870                    false,
2871                    None,
2872                    blockquote_depth,
2873                    pending_style.clone(),
2874                );
2875                pending_style.page_break_before = None;
2876            } else if !current_spans.is_empty() {
2877                // Mirrors the Markdown importer: a hard break splits the
2878                // paragraph into a new block.
2879                djot_push_block(
2880                    &mut elements,
2881                    std::mem::take(&mut current_spans),
2882                    None,
2883                    cur_list_style.clone(),
2884                    cur_list_indent,
2885                    cur_list_prefix.clone(),
2886                    cur_list_suffix.clone(),
2887                    cur_marker.clone(),
2888                    is_code_block,
2889                    code_language.clone(),
2890                    blockquote_depth,
2891                    pending_style.clone(),
2892                );
2893                pending_style.page_break_before = None;
2894            }
2895        }};
2896    }
2897
2898    for event in events {
2899        if skip_depth > 0 {
2900            match event {
2901                E::Start(..) => skip_depth += 1,
2902                E::End(_) => skip_depth -= 1,
2903                _ => {}
2904            }
2905            continue;
2906        }
2907
2908        match event {
2909            // ── Transparent wrappers (unwrap, keep content) ──
2910            E::Start(C::Document, _) | E::End(C::Document) => {}
2911            E::Start(C::Section { .. }, attrs) => {
2912                // A heading's block attributes attach to its enclosing Section,
2913                // not the heading itself; capture them for the heading's flush.
2914                if list_stack.is_empty() {
2915                    pending_style.merge_from(block_attrs_to_style(&attrs, options));
2916                }
2917            }
2918            E::End(C::Section { .. }) => {}
2919            E::Start(C::Div { .. }, _) | E::End(C::Div { .. }) => {}
2920
2921            // ── Blockquote ──
2922            E::Start(C::Blockquote, _) => blockquote_depth += 1,
2923            E::End(C::Blockquote) => blockquote_depth = blockquote_depth.saturating_sub(1),
2924
2925            // ── Lists ──
2926            E::Start(C::List { kind, .. }, _) => {
2927                let (style, prefix, suffix) = match kind {
2928                    ListKind::Unordered(b) | ListKind::Task(b) => {
2929                        (djot_bullet_style(b), String::new(), String::new())
2930                    }
2931                    ListKind::Ordered {
2932                        numbering, style, ..
2933                    } => {
2934                        let (p, s) = djot_ordered_affixes(style);
2935                        (djot_ordered_style(numbering), p, s)
2936                    }
2937                };
2938                list_stack.push((style, prefix, suffix));
2939            }
2940            E::End(C::List { .. }) => {
2941                list_stack.pop();
2942                cur_list_style = None;
2943                cur_marker = None;
2944            }
2945            E::Start(C::ListItem, _) => enter_item!(None),
2946            E::Start(C::TaskListItem { checked }, _) => enter_item!(Some(if checked {
2947                MarkerType::Checked
2948            } else {
2949                MarkerType::Unchecked
2950            })),
2951            E::End(C::ListItem) | E::End(C::TaskListItem { .. }) => {
2952                // Tight item without a wrapping paragraph (defensive flush).
2953                if !current_spans.is_empty() {
2954                    djot_push_block(
2955                        &mut elements,
2956                        std::mem::take(&mut current_spans),
2957                        None,
2958                        cur_list_style.clone(),
2959                        cur_list_indent,
2960                        cur_list_prefix.clone(),
2961                        cur_list_suffix.clone(),
2962                        cur_marker.clone(),
2963                        false,
2964                        None,
2965                        blockquote_depth,
2966                        DjotBlockStyle::default(),
2967                    );
2968                }
2969                cur_list_style = None;
2970                cur_marker = None;
2971            }
2972
2973            // ── Headings, paragraphs, code blocks ──
2974            E::Start(C::Heading { level, .. }, attrs) => {
2975                current_heading = Some(level as i64);
2976                // The block-style attributes live on the enclosing Section;
2977                // merge any placed directly on the heading without clearing them.
2978                pending_style.merge_from(block_attrs_to_style(&attrs, options));
2979            }
2980            E::End(C::Heading { .. }) => {
2981                djot_push_block(
2982                    &mut elements,
2983                    std::mem::take(&mut current_spans),
2984                    current_heading.take(),
2985                    None,
2986                    0,
2987                    String::new(),
2988                    String::new(),
2989                    None,
2990                    false,
2991                    None,
2992                    blockquote_depth,
2993                    std::mem::take(&mut pending_style),
2994                );
2995            }
2996            E::Start(C::Paragraph, attrs) => {
2997                current_heading = None;
2998                // Block attributes only apply to standalone paragraphs;
2999                // list-item paragraphs normalise their styling away (matching
3000                // the exporter).
3001                pending_style = if list_stack.is_empty() {
3002                    block_attrs_to_style(&attrs, options)
3003                } else {
3004                    DjotBlockStyle::default()
3005                };
3006            }
3007            E::End(C::Paragraph) => {
3008                if !current_spans.is_empty() {
3009                    djot_push_block(
3010                        &mut elements,
3011                        std::mem::take(&mut current_spans),
3012                        None,
3013                        cur_list_style.clone(),
3014                        cur_list_indent,
3015                        cur_list_prefix.clone(),
3016                        cur_list_suffix.clone(),
3017                        cur_marker.clone(),
3018                        false,
3019                        None,
3020                        blockquote_depth,
3021                        std::mem::take(&mut pending_style),
3022                    );
3023                }
3024                cur_list_style = None;
3025                cur_marker = None;
3026            }
3027            E::Start(C::CodeBlock { language }, _) => {
3028                is_code_block = true;
3029                code_language = if language.is_empty() {
3030                    None
3031                } else {
3032                    Some(language.to_string())
3033                };
3034            }
3035            E::End(C::CodeBlock { .. }) => {
3036                // Strip the single trailing newline jotdown appends.
3037                if let Some(last) = current_spans.last_mut()
3038                    && last.text.ends_with('\n')
3039                {
3040                    last.text.pop();
3041                }
3042                djot_push_block(
3043                    &mut elements,
3044                    std::mem::take(&mut current_spans),
3045                    None,
3046                    None,
3047                    0,
3048                    String::new(),
3049                    String::new(),
3050                    None,
3051                    true,
3052                    code_language.take(),
3053                    blockquote_depth,
3054                    DjotBlockStyle::default(),
3055                );
3056                is_code_block = false;
3057            }
3058
3059            // ── Tables ──
3060            E::Start(C::Table, _) => {
3061                table_rows.clear();
3062                current_row.clear();
3063                current_cell_spans.clear();
3064                table_header_rows = 0;
3065            }
3066            E::End(C::Table) => {
3067                let own_cells = cells_in_rows(&table_rows);
3068                push_table(
3069                    &mut elements,
3070                    ParsedTable {
3071                        header_rows: table_header_rows,
3072                        rows: std::mem::take(&mut table_rows),
3073                        blockquote_depth,
3074                    },
3075                    own_cells,
3076                );
3077            }
3078            E::Start(C::TableRow { head }, _) => {
3079                row_is_head = head;
3080                current_row.clear();
3081            }
3082            E::End(C::TableRow { .. }) => {
3083                if row_is_head {
3084                    table_header_rows += 1;
3085                }
3086                table_rows.push(std::mem::take(&mut current_row));
3087            }
3088            E::Start(C::TableCell { .. }, _) => {
3089                in_table_cell = true;
3090                current_cell_spans.clear();
3091            }
3092            E::End(C::TableCell { .. }) => {
3093                in_table_cell = false;
3094                current_row.push(ParsedTableCell {
3095                    spans: std::mem::take(&mut current_cell_spans),
3096                });
3097            }
3098
3099            // ── Inline formatting ──
3100            E::Start(C::Strong, _) => bold = true,
3101            E::End(C::Strong) => bold = false,
3102            E::Start(C::Emphasis, _) => italic = true,
3103            E::End(C::Emphasis) => italic = false,
3104            E::Start(C::Verbatim, _) => code = true,
3105            E::End(C::Verbatim) => code = false,
3106            E::Start(C::Superscript, _) => superscript = true,
3107            E::End(C::Superscript) => superscript = false,
3108            E::Start(C::Subscript, _) => subscript = true,
3109            E::End(C::Subscript) => subscript = false,
3110            E::Start(C::Insert, _) => underline = true,
3111            E::End(C::Insert) => underline = false,
3112            E::Start(C::Delete, _) => strikeout = true,
3113            E::End(C::Delete) => strikeout = false,
3114            // Highlight/mark and bare spans have no model field — keep the text.
3115            E::Start(C::Mark, _) | E::End(C::Mark) => {}
3116            E::Start(C::Span, _) | E::End(C::Span) => {}
3117            E::Start(C::Link(dst, _), _) => link_href = Some(dst.to_string()),
3118            E::End(C::Link(..)) => link_href = None,
3119            // Inline images. Djot writes display size as inline attributes
3120            // (`![alt](src){width=800 height=600}`), which jotdown hands over
3121            // on the `Start` event — verified against jotdown 0.10, including
3122            // quoted values and images mid-sentence.
3123            E::Start(C::Image(src, _), attrs) => {
3124                let attr_num = |key: &str| -> i64 {
3125                    attrs
3126                        .get_value(key)
3127                        .map(|v| v.to_string())
3128                        .and_then(|v| v.trim().parse::<i64>().ok())
3129                        .filter(|n| *n > 0)
3130                        .unwrap_or(0)
3131                };
3132                pending_image = Some(ParsedImage {
3133                    // The writer percent-encodes what the lexer would act on in a
3134                    // source; the source is a name, so it is decoded back.
3135                    src: crate::parser_tools::djot_escape::decode_djot_image_source(&src),
3136                    alt: String::new(),
3137                    width: attr_num("width"),
3138                    height: attr_num("height"),
3139                });
3140            }
3141            E::End(C::Image(..)) => {
3142                if let Some(img) = pending_image.take() {
3143                    push_image!(img);
3144                }
3145            }
3146
3147            // ── Footnote definitions ──
3148            //
3149            // The body is ordinary block content, so it is parsed by the
3150            // ordinary machinery: flush whatever inline run is open, note where
3151            // this definition's blocks start, and let them accumulate. `End`
3152            // lifts them back out. Nesting cannot occur — djot has no footnote
3153            // inside a footnote — so one mark suffices where the dropped
3154            // containers below need a depth counter.
3155            E::Start(C::Footnote { label }, _) => {
3156                if !current_spans.is_empty() {
3157                    djot_push_block(
3158                        &mut elements,
3159                        std::mem::take(&mut current_spans),
3160                        None,
3161                        cur_list_style.clone(),
3162                        cur_list_indent,
3163                        cur_list_prefix.clone(),
3164                        cur_list_suffix.clone(),
3165                        cur_marker.clone(),
3166                        false,
3167                        None,
3168                        blockquote_depth,
3169                        DjotBlockStyle::default(),
3170                    );
3171                }
3172                footnote_open = Some((label.to_string(), elements.len()));
3173            }
3174            E::End(C::Footnote { .. }) => {
3175                if !current_spans.is_empty() {
3176                    djot_push_block(
3177                        &mut elements,
3178                        std::mem::take(&mut current_spans),
3179                        None,
3180                        cur_list_style.clone(),
3181                        cur_list_indent,
3182                        cur_list_prefix.clone(),
3183                        cur_list_suffix.clone(),
3184                        cur_marker.clone(),
3185                        false,
3186                        None,
3187                        blockquote_depth,
3188                        DjotBlockStyle::default(),
3189                    );
3190                }
3191                if let Some((label, start)) = footnote_open.take() {
3192                    let blocks: Vec<ParsedBlock> = elements
3193                        .drain(start..)
3194                        .filter_map(|e| match e {
3195                            ParsedElement::Block(b) => Some(b),
3196                            // A table inside a footnote is not representable as
3197                            // note content; its cells would have to become
3198                            // blocks and lose their structure either way.
3199                            _ => None,
3200                        })
3201                        .collect();
3202                    elements.push(ParsedElement::FootnoteDefinition { label, blocks });
3203                }
3204            }
3205
3206            // ── Unrepresentable containers: drop the entire subtree ──
3207            E::Start(
3208                C::Math { .. }
3209                | C::RawBlock { .. }
3210                | C::RawInline { .. }
3211                | C::DescriptionList
3212                | C::DescriptionDetails
3213                | C::DescriptionTerm
3214                | C::Caption
3215                | C::LinkDefinition { .. },
3216                _,
3217            ) => skip_depth = 1,
3218
3219            // ── Text + atoms ──
3220            // A line ending inside a code span is kept by the parser as a line
3221            // break in the span's text, the one a paragraph's text can hold. It is
3222            // read as a hard break is, a new block with the list, quotation or
3223            // heading the line was in, so the model holds no line break inside a
3224            // block. Kept inside the span, the Djot writer saved the code run's
3225            // lines as blocks of their own, which reloaded as two blocks: a document
3226            // that did not settle into the Djot it saves as. Such a span is what an
3227            // older version wrote for a code run holding a line break, a pasted
3228            // `<pre><code>` passage above all, and reading its lines as blocks is how
3229            // that passage reads in the editor.
3230            E::Str(s) if code && !is_code_block && pending_image.is_none() && s.contains('\n') => {
3231                for (i, piece) in s.split('\n').enumerate() {
3232                    if i > 0 {
3233                        line_break!();
3234                    }
3235                    if !piece.is_empty() {
3236                        push_text!(piece);
3237                    }
3238                }
3239            }
3240            E::Str(s) => push_text!(s.as_ref()),
3241            E::Softbreak => push_text!(" "),
3242            E::LeftSingleQuote => push_text!("\u{2018}"),
3243            E::RightSingleQuote => push_text!("\u{2019}"),
3244            E::LeftDoubleQuote => push_text!("\u{201C}"),
3245            E::RightDoubleQuote => push_text!("\u{201D}"),
3246            E::Ellipsis => push_text!("\u{2026}"),
3247            E::EnDash => push_text!("\u{2013}"),
3248            E::EmDash => push_text!("\u{2014}"),
3249            E::NonBreakingSpace => push_text!("\u{00A0}"),
3250            E::Hardbreak => line_break!(),
3251            // A footnote reference. jotdown emits this purely syntactically —
3252            // it never checks that a matching `[^label]:` exists anywhere — so a
3253            // reference whose definition lives outside this document (the normal
3254            // state for a host that owns note bodies itself) arrives here just
3255            // the same, and must survive.
3256            E::FootnoteReference(label) => {
3257                let sp = ParsedSpan {
3258                    text: String::new(),
3259                    bold,
3260                    italic,
3261                    underline,
3262                    strikeout,
3263                    code,
3264                    superscript,
3265                    subscript,
3266                    link_href: link_href.clone(),
3267                    image: None,
3268                    footnote_ref: Some(label.to_string()),
3269                };
3270                if in_table_cell {
3271                    current_cell_spans.push(sp);
3272                } else {
3273                    current_spans.push(sp);
3274                }
3275            }
3276            // Symbols, escapes, blanklines, thematic breaks and dangling block
3277            // attributes carry no representable content.
3278            E::Symbol(_) => {}
3279            E::Escape | E::Blankline => {}
3280            E::ThematicBreak(_) | E::Attributes(_) => {}
3281
3282            // Ends of dropped containers (never reached at skip_depth 0) and any
3283            // future variants.
3284            _ => {}
3285        }
3286    }
3287
3288    // Flush any trailing inline content (defensive — Document End closes blocks).
3289    if !current_spans.is_empty() {
3290        djot_push_block(
3291            &mut elements,
3292            std::mem::take(&mut current_spans),
3293            current_heading.take(),
3294            cur_list_style.clone(),
3295            cur_list_indent,
3296            cur_list_prefix.clone(),
3297            cur_list_suffix.clone(),
3298            cur_marker.clone(),
3299            is_code_block,
3300            code_language.take(),
3301            blockquote_depth,
3302            std::mem::take(&mut pending_style),
3303        );
3304    }
3305
3306    // An empty document still yields a single empty paragraph (matches
3307    // `parse_markdown`).
3308    if elements.is_empty() {
3309        djot_push_block(
3310            &mut elements,
3311            vec![ParsedSpan {
3312                text: String::new(),
3313                ..Default::default()
3314            }],
3315            None,
3316            None,
3317            0,
3318            String::new(),
3319            String::new(),
3320            None,
3321            false,
3322            None,
3323            0,
3324            DjotBlockStyle::default(),
3325        );
3326    }
3327
3328    elements
3329}
3330
3331#[cfg(test)]
3332mod tests {
3333    use super::*;
3334
3335    /// Helper: flatten parse_markdown output to blocks for tests that don't care about tables.
3336    fn parse_markdown_blocks(md: &str) -> Vec<ParsedBlock> {
3337        ParsedElement::flatten_to_blocks(parse_markdown(md))
3338    }
3339
3340    #[test]
3341    fn test_parse_markdown_simple_paragraph() {
3342        let blocks = parse_markdown_blocks("Hello **world**");
3343        assert_eq!(blocks.len(), 1);
3344        assert!(blocks[0].spans.len() >= 2);
3345        // "Hello " is plain, "world" is bold
3346        let plain_span = blocks[0]
3347            .spans
3348            .iter()
3349            .find(|s| s.text.contains("Hello"))
3350            .unwrap();
3351        assert!(!plain_span.bold);
3352        let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3353        assert!(bold_span.bold);
3354    }
3355
3356    #[test]
3357    fn test_parse_markdown_heading() {
3358        let blocks = parse_markdown_blocks("# Title");
3359        assert_eq!(blocks.len(), 1);
3360        assert_eq!(blocks[0].heading_level, Some(1));
3361        assert_eq!(blocks[0].spans[0].text, "Title");
3362    }
3363
3364    #[test]
3365    fn test_parse_markdown_list() {
3366        let blocks = parse_markdown_blocks("- item1\n- item2");
3367        assert!(blocks.len() >= 2);
3368        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3369        assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
3370    }
3371
3372    /// Helper: extract (is_table, blockquote_depth) per element for nesting assertions.
3373    fn element_depths(elements: &[ParsedElement]) -> Vec<(bool, u32)> {
3374        elements
3375            .iter()
3376            .map(|e| match e {
3377                ParsedElement::Block(b) => (false, b.blockquote_depth),
3378                ParsedElement::Table(t) => (true, t.blockquote_depth),
3379                // Definitions carry no blockquote nesting of their own; this
3380                // helper exists for the nesting assertions and never sees one.
3381                ParsedElement::FootnoteDefinition { .. } => (false, 0),
3382            })
3383            .collect()
3384    }
3385
3386    #[test]
3387    fn test_parse_markdown_table_in_blockquote_records_depth() {
3388        let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3389        assert_eq!(element_depths(&elements), vec![(true, 1)]);
3390    }
3391
3392    #[test]
3393    fn test_parse_markdown_text_then_table_in_blockquote() {
3394        let elements = parse_markdown("> Para\n>\n> | a | b |\n> |---|---|\n> | c | d |");
3395        assert_eq!(element_depths(&elements), vec![(false, 1), (true, 1)]);
3396    }
3397
3398    #[test]
3399    fn test_parse_markdown_table_after_blockquote_closes() {
3400        let elements = parse_markdown("> Para\n\n| a | b |\n|---|---|\n| c | d |");
3401        assert_eq!(element_depths(&elements), vec![(false, 1), (true, 0)]);
3402    }
3403
3404    #[test]
3405    fn test_parse_markdown_table_in_nested_blockquote() {
3406        let elements = parse_markdown(">> | a | b |\n>> |---|---|\n>> | c | d |");
3407        assert_eq!(element_depths(&elements), vec![(true, 2)]);
3408    }
3409
3410    #[test]
3411    fn test_parse_markdown_list_in_blockquote_records_depth() {
3412        let elements = parse_markdown("> - item1\n> - item2");
3413        let depths = element_depths(&elements);
3414        assert_eq!(depths, vec![(false, 1), (false, 1)]);
3415        for e in &elements {
3416            if let ParsedElement::Block(b) = e {
3417                assert_eq!(b.list_style, Some(ListStyle::Disc));
3418            }
3419        }
3420    }
3421
3422    #[test]
3423    fn test_parse_html_table_in_blockquote_records_depth() {
3424        let elements = parse_html_elements(
3425            "<blockquote><table><tr><th>A</th></tr><tr><td>x</td></tr></table></blockquote>",
3426        );
3427        assert_eq!(element_depths(&elements), vec![(true, 1)]);
3428    }
3429
3430    #[test]
3431    fn test_parse_html_table_after_blockquote() {
3432        let elements = parse_html_elements(
3433            "<blockquote><p>Para</p></blockquote><table><tr><td>X</td></tr></table>",
3434        );
3435        let depths = element_depths(&elements);
3436        // The blockquote paragraph carries depth 1; the table is outside (depth 0).
3437        assert!(depths.contains(&(false, 1)), "depths: {depths:?}");
3438        assert!(depths.contains(&(true, 0)), "depths: {depths:?}");
3439    }
3440
3441    #[test]
3442    fn test_flatten_to_blocks_propagates_blockquote_depth() {
3443        let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3444        let blocks = ParsedElement::flatten_to_blocks(elements);
3445        assert!(!blocks.is_empty());
3446        for b in &blocks {
3447            assert_eq!(b.blockquote_depth, 1);
3448        }
3449    }
3450
3451    #[test]
3452    fn test_parse_html_simple() {
3453        let blocks = parse_html("<p>Hello <b>world</b></p>");
3454        assert_eq!(blocks.len(), 1);
3455        assert!(blocks[0].spans.len() >= 2);
3456        let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3457        assert!(bold_span.bold);
3458    }
3459
3460    #[test]
3461    fn test_parse_html_multiple_paragraphs() {
3462        let blocks = parse_html("<p>A</p><p>B</p>");
3463        assert_eq!(blocks.len(), 2);
3464    }
3465
3466    #[test]
3467    fn test_parse_html_heading() {
3468        let blocks = parse_html("<h2>Subtitle</h2>");
3469        assert_eq!(blocks.len(), 1);
3470        assert_eq!(blocks[0].heading_level, Some(2));
3471    }
3472
3473    #[test]
3474    fn test_parse_html_list() {
3475        let blocks = parse_html("<ul><li>one</li><li>two</li></ul>");
3476        assert!(blocks.len() >= 2);
3477        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3478    }
3479
3480    #[test]
3481    fn test_parse_markdown_code_block() {
3482        let blocks = parse_markdown_blocks("```\nfn main() {}\n```");
3483        assert_eq!(blocks.len(), 1);
3484        assert!(blocks[0].is_code_block);
3485        assert!(blocks[0].spans[0].code);
3486        // pulldown-cmark appends a trailing \n to code block text — verify it's stripped
3487        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3488        assert_eq!(
3489            text, "fn main() {}",
3490            "code block text should not have trailing newline"
3491        );
3492    }
3493
3494    #[test]
3495    fn test_parse_markdown_nested_formatting() {
3496        let blocks = parse_markdown_blocks("***bold italic***");
3497        assert_eq!(blocks.len(), 1);
3498        let span = &blocks[0].spans[0];
3499        assert!(span.bold);
3500        assert!(span.italic);
3501    }
3502
3503    #[test]
3504    fn test_parse_markdown_link() {
3505        let blocks = parse_markdown_blocks("[click](http://example.com)");
3506        assert_eq!(blocks.len(), 1);
3507        let span = &blocks[0].spans[0];
3508        assert_eq!(span.text, "click");
3509        assert_eq!(span.link_href, Some("http://example.com".to_string()));
3510    }
3511
3512    #[test]
3513    fn test_parse_markdown_empty() {
3514        let blocks = parse_markdown_blocks("");
3515        assert_eq!(blocks.len(), 1);
3516        assert!(blocks[0].spans[0].text.is_empty());
3517    }
3518
3519    #[test]
3520    fn test_parse_html_empty() {
3521        let blocks = parse_html("");
3522        assert_eq!(blocks.len(), 1);
3523        assert!(blocks[0].spans[0].text.is_empty());
3524    }
3525
3526    #[test]
3527    fn test_parse_html_nested_formatting() {
3528        let blocks = parse_html("<p><b><i>bold italic</i></b></p>");
3529        assert_eq!(blocks.len(), 1);
3530        let span = &blocks[0].spans[0];
3531        assert!(span.bold);
3532        assert!(span.italic);
3533    }
3534
3535    #[test]
3536    fn test_parse_html_link() {
3537        let blocks = parse_html("<p><a href=\"http://example.com\">click</a></p>");
3538        assert_eq!(blocks.len(), 1);
3539        let span = &blocks[0].spans[0];
3540        assert_eq!(span.text, "click");
3541        assert_eq!(span.link_href, Some("http://example.com".to_string()));
3542    }
3543
3544    #[test]
3545    fn test_parse_html_ordered_list() {
3546        let blocks = parse_html("<ol><li>first</li><li>second</li></ol>");
3547        assert!(blocks.len() >= 2);
3548        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3549    }
3550
3551    #[test]
3552    fn test_parse_markdown_ordered_list() {
3553        let blocks = parse_markdown_blocks("1. first\n2. second");
3554        assert!(blocks.len() >= 2);
3555        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3556    }
3557
3558    #[test]
3559    fn test_parse_html_blockquote_nested() {
3560        let blocks = parse_html("<p>before</p><blockquote>quoted</blockquote><p>after</p>");
3561        assert!(blocks.len() >= 3);
3562    }
3563
3564    #[test]
3565    fn test_parse_block_styles_line_height() {
3566        let styles = parse_block_styles("line-height: 1.5");
3567        assert_eq!(styles.line_height, Some(1500));
3568    }
3569
3570    #[test]
3571    fn test_parse_block_styles_direction_rtl() {
3572        let styles = parse_block_styles("direction: rtl");
3573        assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3574    }
3575
3576    #[test]
3577    fn test_parse_block_styles_background_color() {
3578        let styles = parse_block_styles("background-color: #ff0000");
3579        assert_eq!(styles.background_color, Some("#ff0000".to_string()));
3580    }
3581
3582    #[test]
3583    fn test_parse_block_styles_white_space_pre() {
3584        let styles = parse_block_styles("white-space: pre");
3585        assert_eq!(styles.non_breakable_lines, Some(true));
3586    }
3587
3588    #[test]
3589    fn test_parse_block_styles_multiple() {
3590        let styles = parse_block_styles("line-height: 2.0; direction: rtl; background-color: blue");
3591        assert_eq!(styles.line_height, Some(2000));
3592        assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3593        assert_eq!(styles.background_color, Some("blue".to_string()));
3594    }
3595
3596    #[test]
3597    fn test_parse_html_block_styles_extracted() {
3598        let blocks = parse_html(
3599            r#"<p style="line-height: 1.5; direction: rtl; background-color: #ccc">text</p>"#,
3600        );
3601        assert_eq!(blocks.len(), 1);
3602        assert_eq!(blocks[0].line_height, Some(1500));
3603        assert_eq!(blocks[0].direction, Some(TextDirection::RightToLeft));
3604        assert_eq!(blocks[0].background_color, Some("#ccc".to_string()));
3605    }
3606
3607    /// The shape every word processor publishes on the clipboard: a newline
3608    /// after the opening tag, the prose hard-wrapped inside it, and a newline
3609    /// between every pair of blocks. None of it is text.
3610    #[test]
3611    fn test_parse_html_collapses_exporter_line_wrapping() {
3612        let blocks = parse_html(
3613            "<body>\n<p style=\"line-height: 200%\">\nfirst line wrapped\nhere</p>\n\
3614             <p>\nsecond para</p>\n</body>",
3615        );
3616        let texts: Vec<String> = blocks
3617            .iter()
3618            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
3619            .collect();
3620        assert_eq!(texts, vec!["first line wrapped here", "second para"]);
3621    }
3622
3623    /// The reported bug: pasting a whole LibreOffice book ended in one block
3624    /// holding every newline the exporter had written between its paragraphs —
3625    /// four thousand blank lines under the last sentence.
3626    #[test]
3627    fn test_parse_html_drops_whitespace_between_blocks() {
3628        let blocks = parse_html("<html>\n<body>\n<p>a</p>\n<p>b</p>\n</body>\n</html>");
3629        assert_eq!(blocks.len(), 2, "no block for the whitespace between them");
3630        assert!(
3631            blocks
3632                .iter()
3633                .all(|b| b.spans.iter().all(|s| !s.text.contains('\n'))),
3634            "no block keeps a literal newline"
3635        );
3636    }
3637
3638    /// A run split across inline elements is still one run, and it collapses to
3639    /// one space rather than disappearing between them.
3640    #[test]
3641    fn test_parse_html_collapses_across_span_boundaries() {
3642        let blocks = parse_html("<p>  <b>bold</b>\n  <i>italic</i>  </p>");
3643        assert_eq!(blocks.len(), 1);
3644        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3645        assert_eq!(text, "bold italic");
3646    }
3647
3648    /// U+00A0 is content, not layout — the space holding "Attention&nbsp;:"
3649    /// together in French must survive a paste.
3650    #[test]
3651    fn test_parse_html_keeps_no_break_space() {
3652        let blocks = parse_html("<p>\nAttention&nbsp;: ici</p>");
3653        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3654        assert_eq!(text, "Attention\u{a0}: ici");
3655    }
3656
3657    /// An empty paragraph is a paragraph. Only the whitespace *between* blocks
3658    /// goes; a block the writer opened stays, however little is in it.
3659    #[test]
3660    fn test_parse_html_keeps_empty_paragraph() {
3661        let blocks = parse_html("<body>\n<p>a</p>\n<p> </p>\n<p>b</p>\n</body>");
3662        assert_eq!(blocks.len(), 3);
3663        let text: String = blocks[1].spans.iter().map(|s| s.text.as_str()).collect();
3664        assert_eq!(text, "");
3665    }
3666
3667    /// `<pre>` and `white-space: pre` mean the source's spacing *is* the
3668    /// content. `nowrap` does not: it forbids wrapping and collapses runs like
3669    /// any other block, which is why it is not in the same test.
3670    #[test]
3671    fn test_parse_html_preserves_whitespace_in_pre() {
3672        // The newline straight after `<pre>` is the one the HTML parser itself
3673        // drops, per spec; every newline after that is content.
3674        let blocks = parse_html("<pre>\nfn main() {\n    let x = 1;\n}</pre>");
3675        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3676        assert_eq!(text, "fn main() {\n    let x = 1;\n}");
3677
3678        // Outside a code block a line break is a new block, and each line keeps its
3679        // spaces.
3680        let styled = parse_html("<p style=\"white-space: pre-wrap\">a\n  b</p>");
3681        let texts: Vec<String> = styled
3682            .iter()
3683            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
3684            .collect();
3685        assert_eq!(texts, ["a", "  b"]);
3686
3687        let nowrap = parse_html("<p style=\"white-space: nowrap\">a\n  b</p>");
3688        let text: String = nowrap[0].spans.iter().map(|s| s.text.as_str()).collect();
3689        assert_eq!(text, "a b");
3690    }
3691
3692    /// Table cells are blocks too, and an exporter indents them just as
3693    /// generously as it indents paragraphs.
3694    #[test]
3695    fn test_parse_html_collapses_table_cell_whitespace() {
3696        let elements = parse_html_elements(
3697            "<table>\n<tr>\n<td>\n  one\n</td>\n<td>\n  two\n</td>\n</tr>\n</table>",
3698        );
3699        let table = elements
3700            .iter()
3701            .find_map(|e| match e {
3702                ParsedElement::Table(t) => Some(t),
3703                _ => None,
3704            })
3705            .expect("a table");
3706        let cells: Vec<String> = table.rows[0]
3707            .iter()
3708            .map(|c| c.spans.iter().map(|s| s.text.as_str()).collect())
3709            .collect();
3710        assert_eq!(cells, vec!["one", "two"]);
3711    }
3712
3713    #[test]
3714    fn test_parse_html_white_space_pre() {
3715        let blocks = parse_html(r#"<p style="white-space: pre">code</p>"#);
3716        assert_eq!(blocks.len(), 1);
3717        assert_eq!(blocks[0].non_breakable_lines, Some(true));
3718    }
3719
3720    #[test]
3721    fn test_parse_html_no_styles_returns_none() {
3722        let blocks = parse_html("<p>plain</p>");
3723        assert_eq!(blocks.len(), 1);
3724        assert_eq!(blocks[0].line_height, None);
3725        assert_eq!(blocks[0].direction, None);
3726        assert_eq!(blocks[0].background_color, None);
3727        assert_eq!(blocks[0].non_breakable_lines, None);
3728    }
3729
3730    #[test]
3731    fn test_parse_markdown_nested_list_indent() {
3732        let md = "- top\n  - nested\n    - deep";
3733        let blocks = parse_markdown_blocks(md);
3734        assert_eq!(blocks.len(), 3);
3735        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3736        assert_eq!(blocks[0].list_indent, 0);
3737        assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
3738        assert_eq!(blocks[1].list_indent, 1);
3739        assert_eq!(blocks[2].list_style, Some(ListStyle::Disc));
3740        assert_eq!(blocks[2].list_indent, 2);
3741    }
3742
3743    #[test]
3744    fn test_parse_markdown_nested_ordered_list_indent() {
3745        let md = "1. first\n   1. nested\n   2. nested2";
3746        let blocks = parse_markdown_blocks(md);
3747        assert_eq!(blocks.len(), 3);
3748        assert_eq!(blocks[0].list_indent, 0);
3749        assert_eq!(blocks[1].list_indent, 1);
3750        assert_eq!(blocks[2].list_indent, 1);
3751    }
3752
3753    #[test]
3754    fn test_parse_html_nested_list_indent() {
3755        let html = "<ul><li>top</li><ul><li>nested</li></ul></ul>";
3756        let blocks = parse_html(html);
3757        assert!(blocks.len() >= 2);
3758        assert_eq!(blocks[0].list_indent, 0);
3759        assert_eq!(blocks[1].list_indent, 1);
3760    }
3761
3762    #[test]
3763    fn test_parse_markdown_table() {
3764        let md = "| A | B |\n|---|---|\n| 1 | 2 |";
3765        let elements = parse_markdown(md);
3766        assert_eq!(elements.len(), 1);
3767        match &elements[0] {
3768            ParsedElement::Table(table) => {
3769                assert_eq!(table.header_rows, 1);
3770                assert_eq!(table.rows.len(), 2); // 1 header + 1 body
3771                // Header row
3772                assert_eq!(table.rows[0].len(), 2);
3773                assert_eq!(table.rows[0][0].spans[0].text, "A");
3774                assert_eq!(table.rows[0][1].spans[0].text, "B");
3775                // Body row
3776                assert_eq!(table.rows[1].len(), 2);
3777                assert_eq!(table.rows[1][0].spans[0].text, "1");
3778                assert_eq!(table.rows[1][1].spans[0].text, "2");
3779            }
3780            _ => panic!("Expected ParsedElement::Table"),
3781        }
3782    }
3783
3784    #[test]
3785    fn test_parse_markdown_table_with_formatting() {
3786        let md = "| **bold** | `code` | *italic* |\n|---|---|---|\n| ~~strike~~ | plain | [link](http://x.com) |";
3787        let elements = parse_markdown(md);
3788        assert_eq!(elements.len(), 1);
3789        match &elements[0] {
3790            ParsedElement::Table(table) => {
3791                assert_eq!(table.rows.len(), 2);
3792                // Header: bold cell
3793                assert!(table.rows[0][0].spans[0].bold);
3794                // Header: code cell
3795                assert!(table.rows[0][1].spans[0].code);
3796                // Header: italic cell
3797                assert!(table.rows[0][2].spans[0].italic);
3798                // Body: strikeout cell
3799                assert!(table.rows[1][0].spans[0].strikeout);
3800                // Body: link cell
3801                assert_eq!(
3802                    table.rows[1][2].spans[0].link_href,
3803                    Some("http://x.com".to_string())
3804                );
3805            }
3806            _ => panic!("Expected ParsedElement::Table"),
3807        }
3808    }
3809
3810    #[test]
3811    fn test_parse_markdown_mixed_content_with_table() {
3812        let md = "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter";
3813        let elements = parse_markdown(md);
3814        assert_eq!(elements.len(), 3);
3815        assert!(matches!(&elements[0], ParsedElement::Block(_)));
3816        assert!(matches!(&elements[1], ParsedElement::Table(_)));
3817        assert!(matches!(&elements[2], ParsedElement::Block(_)));
3818    }
3819}
3820
3821#[cfg(test)]
3822mod djot_tests {
3823    use super::*;
3824    use crate::entities::MarkerType;
3825
3826    fn blocks(d: &str) -> Vec<ParsedBlock> {
3827        ParsedElement::flatten_to_blocks(parse_djot(d, &DjotImportOptions::default()))
3828    }
3829
3830    fn first_span_with(b: &ParsedBlock, pred: impl Fn(&ParsedSpan) -> bool) -> &ParsedSpan {
3831        b.spans.iter().find(|s| pred(s)).expect("span not found")
3832    }
3833
3834    #[test]
3835    fn paragraph_bold_italic() {
3836        let b = blocks("normal *bold* _italic_");
3837        assert_eq!(b.len(), 1);
3838        assert!(first_span_with(&b[0], |s| s.text == "bold").bold);
3839        assert!(first_span_with(&b[0], |s| s.text == "italic").italic);
3840    }
3841
3842    #[test]
3843    fn heading_levels() {
3844        assert_eq!(blocks("# H1")[0].heading_level, Some(1));
3845        assert_eq!(blocks("### H3")[0].heading_level, Some(3));
3846        assert_eq!(blocks("###### H6")[0].heading_level, Some(6));
3847    }
3848
3849    #[test]
3850    fn unordered_bullet_styles_are_distinct() {
3851        assert_eq!(blocks("- a")[0].list_style, Some(ListStyle::Disc));
3852        assert_eq!(blocks("* a")[0].list_style, Some(ListStyle::Circle));
3853        assert_eq!(blocks("+ a")[0].list_style, Some(ListStyle::Square));
3854    }
3855
3856    #[test]
3857    fn ordered_delimiters() {
3858        let period = blocks("1. a");
3859        assert_eq!(period[0].list_style, Some(ListStyle::Decimal));
3860        assert_eq!(period[0].list_prefix, "");
3861        assert_eq!(period[0].list_suffix, ".");
3862
3863        let paren = blocks("1) a");
3864        assert_eq!(paren[0].list_suffix, ")");
3865        assert_eq!(paren[0].list_prefix, "");
3866
3867        let paren_paren = blocks("(1) a");
3868        assert_eq!(paren_paren[0].list_prefix, "(");
3869        assert_eq!(paren_paren[0].list_suffix, ")");
3870    }
3871
3872    #[test]
3873    fn task_list_markers() {
3874        let b = blocks("- [ ] a\n- [x] b");
3875        assert_eq!(b.len(), 2);
3876        assert_eq!(b[0].marker, Some(MarkerType::Unchecked));
3877        assert_eq!(b[1].marker, Some(MarkerType::Checked));
3878    }
3879
3880    #[test]
3881    fn code_block_with_language() {
3882        let b = blocks("```rust\nfn main() {}\n```");
3883        assert_eq!(b.len(), 1);
3884        assert!(b[0].is_code_block);
3885        assert_eq!(b[0].code_language.as_deref(), Some("rust"));
3886        let text: String = b[0].spans.iter().map(|s| s.text.as_str()).collect();
3887        assert_eq!(text, "fn main() {}");
3888    }
3889
3890    #[test]
3891    fn link_href() {
3892        let b = blocks("[text](http://example.com)");
3893        let s = first_span_with(&b[0], |s| s.text == "text");
3894        assert_eq!(s.link_href.as_deref(), Some("http://example.com"));
3895    }
3896
3897    #[test]
3898    fn superscript_subscript() {
3899        assert!(first_span_with(&blocks("a^b^")[0], |s| s.text == "b").superscript);
3900        assert!(first_span_with(&blocks("a~b~")[0], |s| s.text == "b").subscript);
3901    }
3902
3903    #[test]
3904    fn delete_insert_verbatim() {
3905        assert!(first_span_with(&blocks("{-x-}")[0], |s| s.text == "x").strikeout);
3906        assert!(first_span_with(&blocks("{+x+}")[0], |s| s.text == "x").underline);
3907        assert!(first_span_with(&blocks("`x`")[0], |s| s.text == "x").code);
3908    }
3909
3910    #[test]
3911    fn blockquote_depth() {
3912        let els = parse_djot("> quoted", &DjotImportOptions::default());
3913        match &els[0] {
3914            ParsedElement::Block(b) => assert_eq!(b.blockquote_depth, 1),
3915            _ => panic!("expected block"),
3916        }
3917    }
3918
3919    #[test]
3920    fn nested_list_indent() {
3921        // Djot nests a sub-list only when a blank line separates it from the
3922        // parent item and it is indented to the parent's content column
3923        // (2 spaces per level). Without the blank line the markers fold into
3924        // the paragraph as lazy continuation.
3925        let b = blocks("- a\n\n  - b\n\n    - c");
3926        assert_eq!(b.len(), 3);
3927        assert_eq!(b[0].list_indent, 0);
3928        assert_eq!(b[1].list_indent, 1);
3929        assert_eq!(b[2].list_indent, 2);
3930    }
3931
3932    #[test]
3933    fn table_parsed_as_table() {
3934        let els = parse_djot(
3935            "| a | b |\n|---|---|\n| c | d |",
3936            &DjotImportOptions::default(),
3937        );
3938        assert_eq!(els.len(), 1);
3939        match &els[0] {
3940            ParsedElement::Table(t) => {
3941                assert_eq!(t.header_rows, 1);
3942                assert_eq!(t.rows.len(), 2);
3943                assert_eq!(t.rows[0][0].spans[0].text, "a");
3944                assert_eq!(t.rows[1][1].spans[0].text, "d");
3945            }
3946            _ => panic!("expected table"),
3947        }
3948    }
3949
3950    #[test]
3951    fn smart_punctuation_normalised_to_unicode() {
3952        let text: String = blocks("a... b---c")[0]
3953            .spans
3954            .iter()
3955            .map(|s| s.text.as_str())
3956            .collect();
3957        assert!(text.contains('\u{2026}'), "ellipsis: {text:?}");
3958        assert!(text.contains('\u{2014}'), "em dash: {text:?}");
3959    }
3960
3961    #[test]
3962    fn unrepresentable_constructs_dropped_without_leaking_text() {
3963        // Thematic break between two paragraphs: no extra block, no stray text.
3964        let b = blocks("para1\n\n---\n\npara2");
3965        assert_eq!(b.len(), 2);
3966        assert_eq!(
3967            b[0].spans
3968                .iter()
3969                .map(|s| s.text.as_str())
3970                .collect::<String>(),
3971            "para1"
3972        );
3973        assert_eq!(
3974            b[1].spans
3975                .iter()
3976                .map(|s| s.text.as_str())
3977                .collect::<String>(),
3978            "para2"
3979        );
3980
3981        // Fenced div is unwrapped: its content survives, the fence does not.
3982        let d = blocks(":::\ninside\n:::");
3983        let joined: String = d
3984            .iter()
3985            .flat_map(|b| b.spans.iter())
3986            .map(|s| s.text.as_str())
3987            .collect();
3988        assert_eq!(joined, "inside");
3989
3990        // Inline math content is dropped, surrounding text kept.
3991        let m = blocks("before $`E=mc^2` after");
3992        let joined: String = m
3993            .iter()
3994            .flat_map(|b| b.spans.iter())
3995            .map(|s| s.text.as_str())
3996            .collect();
3997        assert!(joined.contains("before"), "{joined:?}");
3998        assert!(joined.contains("after"), "{joined:?}");
3999        assert!(!joined.contains("E=mc"), "math leaked: {joined:?}");
4000    }
4001
4002    #[test]
4003    fn empty_document_yields_one_empty_block() {
4004        let b = blocks("");
4005        assert_eq!(b.len(), 1);
4006        assert!(b[0].spans.iter().all(|s| s.text.is_empty()));
4007    }
4008
4009    #[test]
4010    fn block_attributes_parse_into_block() {
4011        let b = blocks(
4012            "{alignment=center line_height=1500 direction=rtl non_breakable_lines=true background_color=\"#ff0000\"}\nhello",
4013        );
4014        assert_eq!(b.len(), 1);
4015        assert_eq!(b[0].alignment, Some(Alignment::Center));
4016        assert_eq!(b[0].line_height, Some(1500));
4017        assert_eq!(b[0].direction, Some(TextDirection::RightToLeft));
4018        assert_eq!(b[0].non_breakable_lines, Some(true));
4019        assert_eq!(b[0].background_color, Some("#ff0000".to_string()));
4020    }
4021
4022    #[test]
4023    fn spacing_block_attributes_parse_into_block() {
4024        // `top_margin` / `text_indent` let one block override the document-wide
4025        // paragraph spacing and first-line indent — what a scene break needs for
4026        // the paragraph that follows it.
4027        let b = blocks("{top_margin=24 text_indent=0}\nhello");
4028        assert_eq!(b.len(), 1);
4029        assert_eq!(b[0].top_margin, Some(24));
4030        assert_eq!(b[0].text_indent, Some(0));
4031    }
4032
4033    #[test]
4034    fn a_zero_text_indent_is_distinct_from_an_absent_one() {
4035        // The whole point of the attribute: `Some(0)` means "explicitly no
4036        // indent", which must not collapse to `None` ("use the document
4037        // default") — otherwise a scene break could not suppress the indent.
4038        let explicit = blocks("{text_indent=0}\nhello");
4039        let absent = blocks("hello");
4040        assert_eq!(explicit[0].text_indent, Some(0));
4041        assert_eq!(absent[0].text_indent, None);
4042        assert!(!explicit[0].is_inline_only());
4043        assert!(absent[0].is_inline_only());
4044    }
4045
4046    #[test]
4047    fn spacing_block_attributes_respect_import_options() {
4048        let src = "{top_margin=24 text_indent=0}\nhello";
4049        let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4050        assert_eq!(b[0].top_margin, None);
4051        assert_eq!(b[0].text_indent, None);
4052    }
4053
4054    #[test]
4055    fn block_attributes_on_heading() {
4056        let b = blocks("{alignment=right}\n# Title");
4057        assert_eq!(b[0].heading_level, Some(1));
4058        assert_eq!(b[0].alignment, Some(Alignment::Right));
4059    }
4060
4061    #[test]
4062    fn block_attributes_respect_import_options() {
4063        // With every optional attribute disabled, the `{…}` block attributes are
4064        // parsed and discarded — only the core paragraph survives.
4065        let src = "{alignment=center line_height=1500}\nhello";
4066        let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4067        assert_eq!(b[0].alignment, None);
4068        assert_eq!(b[0].line_height, None);
4069        assert_eq!(
4070            b[0].spans
4071                .iter()
4072                .map(|s| s.text.as_str())
4073                .collect::<String>(),
4074            "hello"
4075        );
4076    }
4077
4078    #[test]
4079    fn list_item_block_attributes_are_dropped() {
4080        // Block attributes only bind to standalone paragraphs/headings; a list
4081        // item normalises them away (symmetric with the exporter).
4082        let b = blocks("{alignment=center}\n- item");
4083        assert!(b.iter().all(|blk| blk.alignment.is_none()));
4084    }
4085
4086    #[test]
4087    fn unknown_alignment_value_is_ignored() {
4088        let b = blocks("{alignment=sideways}\nhello");
4089        assert_eq!(b[0].alignment, None);
4090    }
4091}
4092
4093// ─── Cheap plain-text extraction ─────────────────────────────────────
4094
4095/// The `U+FFFC OBJECT REPLACEMENT CHARACTER` that stands for a table in the document's
4096/// text.
4097///
4098/// A table is not prose, but it *occupies a position* in the flow: the import mirrors this
4099/// single sentinel into the rope where the table sits
4100/// (`rope_helpers::rope_append_table_anchor`), then the cells as ordinary blocks. Anything
4101/// reconstructing the text a search runs against has to reproduce it, or every offset after
4102/// the first table is short by the two characters (the sentinel and its separator) that the
4103/// document really holds there.
4104pub const TABLE_ANCHOR: &str = "\u{FFFC}";
4105
4106/// One span's contribution to the addressable text.
4107///
4108/// An inline object carries no prose but **does** occupy one `U+FFFC` in the
4109/// document (`format_runs_from_spans` mirrors it there), so it has to occupy one
4110/// here too. Leaving it out makes this string shorter than the document it
4111/// claims to be byte-identical to, and every offset past the object — every
4112/// search hit, every comment anchor — lands a character early.
4113fn span_prose(span: &ParsedSpan, out: &mut String) {
4114    if span.image.is_some() || span.footnote_ref.is_some() {
4115        out.push('\u{FFFC}');
4116        return;
4117    }
4118    out.push_str(&span.text);
4119}
4120
4121fn block_prose(block: &ParsedBlock) -> String {
4122    let mut prose = String::new();
4123    for span in &block.spans {
4124        span_prose(span, &mut prose);
4125    }
4126    prose
4127}
4128
4129fn cell_prose(cell: &ParsedTableCell) -> String {
4130    let mut prose = String::new();
4131    for span in &cell.spans {
4132        span_prose(span, &mut prose);
4133    }
4134    prose
4135}
4136
4137/// The prose of a Djot document, with no entities, no store, and no threads.
4138///
4139/// [`parse_djot`] and [`ParsedElement::flatten_to_blocks`] were both already `pub`;
4140/// nothing chained them. This does, and that is the whole trick: it stops at the
4141/// *parse*, where a full import goes on to create a `Block` entity per paragraph, list
4142/// item and table cell, mirror each into the rope, and write its format runs.
4143///
4144/// # Why a project-wide search needs this
4145///
4146/// A host app searching a manuscript must ask "does this scene contain that word" of
4147/// **thousands** of Djot rows, on every keystroke. Doing that by importing each one into
4148/// a document is not a slow feature, it is a frozen app.
4149///
4150/// And searching the Djot *source* instead — the tempting shortcut — is simply wrong:
4151/// the source is markup. `http` matches inside a link's URL, `*` matches an emphasis
4152/// marker, and an occurrence count taken from the source does not agree with what a
4153/// replace re-derives inside the parsed document. Where a replace guards itself with
4154/// "the text moved under me, skip this field", a count taken from markup makes that
4155/// guard fire on perfectly good rows.
4156///
4157/// # The contract
4158///
4159/// The result is **byte-identical to the text the document searches** for the same Djot:
4160/// each block's spans concatenated, blocks joined by a single `\n`, and a table announced
4161/// by its [`TABLE_ANCHOR`] sentinel — exactly the string the import mirrors into the rope
4162/// and that `build_full_text_via_store` recomposes. So an offset found here is an offset
4163/// the document agrees with. On a live document the same string is served by
4164/// `TextDocument::to_addressable_text()`, which reads it off the search plumbing itself.
4165///
4166/// A property in `djot_roundtrip_tests` pins that across the whole generated feature set;
4167/// without it this would be a second, silently-diverging definition of "the text".
4168///
4169/// **Known exception: a footnote *definition*'s body.** This function treats it as out of
4170/// flow and omits it — matching `character_count()`, which does not count it — but the
4171/// live document mirrors its blocks into the rope, so today an in-document search DOES
4172/// run over the note's body and `to_addressable_text()` includes it. The live document
4173/// holds every body after the main text, so this string is the start of that one, and
4174/// every offset of the main text agrees between the two; they differ by the bodies at
4175/// the end. Which side is wrong is an open product question (should search see notes?),
4176/// pinned as current behaviour in
4177/// `addressable_text_tests::footnote_bodies_are_searched_in_the_live_document`.
4178///
4179/// ⚠ It is **not** the same as `TextDocument::to_plain_text()`. That is the human-readable
4180/// *export* — same prose, same order, but with every object anchor omitted, so its offsets
4181/// drift by two characters per preceding table. The authority is what a search sees,
4182/// because that is what a replace edits.
4183/// (`to_plain_text` once also *ordered* a blockquote's prose differently; that was a bug,
4184/// fixed and pinned by `plain_text_order_tests` — see
4185/// `claude_reviews/text-document-plain-text-ordering.md`. The anchors are now the two
4186/// views' only difference.)
4187pub fn djot_to_plain_text(djot: &str, options: &DjotImportOptions) -> String {
4188    // Deliberately NOT `ParsedElement::flatten_to_blocks`: that helper drops a table's
4189    // anchor and yields only its cells, which would silently shift every offset in a
4190    // document containing a table by the two characters the document actually holds there.
4191    let elements = parse_djot(djot, options);
4192
4193    // Sized from the source: the prose is always shorter than the markup that carries it,
4194    // so this allocates once and never grows.
4195    let mut out = String::with_capacity(djot.len());
4196
4197    // The separator is decided by "is this the first block", NOT by "is the output still
4198    // empty". An EMPTY block (an empty code fence, say) is still a block: the document
4199    // holds an empty line for it, and an emptiness test would swallow both the block and
4200    // its separator, shifting every offset after it by one.
4201    let mut first = true;
4202    let push = |text: &str, out: &mut String, first: &mut bool| {
4203        if *first {
4204            *first = false;
4205        } else {
4206            out.push('\n');
4207        }
4208        out.push_str(text);
4209    };
4210
4211    for element in &elements {
4212        match element {
4213            ParsedElement::Block(block) => {
4214                push(&block_prose(block), &mut out, &mut first);
4215            }
4216            // A note's body is **out of flow**: it is not laid out where its
4217            // definition was written, it is not part of a copied fragment, and
4218            // the document does not count its characters. So it is not part of
4219            // the addressable text either — which is the one thing that has to
4220            // stay true here, since `character_count()` and this string are
4221            // compared directly.
4222            //
4223            // Its blocks do live in the rope, because they have to live
4224            // somewhere; they are simply not addressable prose.
4225            ParsedElement::FootnoteDefinition { .. } => {}
4226            ParsedElement::Table(table) => {
4227                // The import mirrors a table into the rope as a lone `U+FFFC` sentinel
4228                // followed by its cells, one per block (`rope_append_table_anchor`). The
4229                // sentinel occupies a real position in the text the document searches, so
4230                // it has to occupy one here too — otherwise every offset after a table is
4231                // short by two characters, and a snippet taken from this string would be
4232                // sliced in the wrong place.
4233                push(TABLE_ANCHOR, &mut out, &mut first);
4234                // A row shorter than the widest holds empty cells in the document, one
4235                // position each: the import completes it (see `ParsedTable::column_count`).
4236                let columns = table.column_count();
4237                for row in &table.rows {
4238                    for column in 0..columns {
4239                        let prose = row.get(column).map(cell_prose).unwrap_or_default();
4240                        push(&prose, &mut out, &mut first);
4241                    }
4242                }
4243            }
4244        }
4245    }
4246    out
4247}
4248
4249#[cfg(test)]
4250mod html_footnote_tests {
4251    use super::*;
4252
4253    /// The reason this exists at all: a producer cannot smuggle a reference
4254    /// through as text, because the Djot escaper neutralises `[`, `^` and `]`.
4255    /// Pinned here so the escaping rule and this workaround cannot drift apart.
4256    #[test]
4257    fn a_literal_reference_in_text_would_be_escaped_into_prose() {
4258        let blocks = parse_html("<p>The ferry.[^1]</p>");
4259        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4260        assert!(
4261            text.contains("[^1]"),
4262            "the parser keeps it as characters: {text:?}"
4263        );
4264        assert!(
4265            blocks[0].spans.iter().all(|s| s.footnote_ref.is_none()),
4266            "text is text — it must not become a reference by accident"
4267        );
4268    }
4269
4270    #[test]
4271    fn an_attributed_element_becomes_a_real_footnote_reference() {
4272        let blocks = parse_html(
4273            r#"<p>The ferry was late.<sup data-footnote-ref="3"></sup> She waited.</p>"#,
4274        );
4275        let refs: Vec<&str> = blocks
4276            .iter()
4277            .flat_map(|b| b.spans.iter())
4278            .filter_map(|s| s.footnote_ref.as_deref())
4279            .collect();
4280        assert_eq!(refs, vec!["3"]);
4281
4282        // And it carries no text of its own, so it cannot be counted as words or
4283        // matched by a search.
4284        let marker = blocks[0]
4285            .spans
4286            .iter()
4287            .find(|s| s.footnote_ref.is_some())
4288            .expect("the reference span");
4289        assert!(marker.text.is_empty(), "{marker:?}");
4290    }
4291
4292    /// The attribute is the contract, not the tag: a producer may use whatever
4293    /// element renders sensibly in a browser.
4294    #[test]
4295    fn any_element_carrying_the_attribute_works() {
4296        for html in [
4297            r#"<p>a<sup data-footnote-ref="1"></sup></p>"#,
4298            r#"<p>a<span data-footnote-ref="1"></span></p>"#,
4299            r##"<p>a<a data-footnote-ref="1" href="#fn1"></a></p>"##,
4300        ] {
4301            let blocks = parse_html(html);
4302            assert_eq!(
4303                blocks
4304                    .iter()
4305                    .flat_map(|b| b.spans.iter())
4306                    .filter_map(|s| s.footnote_ref.as_deref())
4307                    .collect::<Vec<_>>(),
4308                vec!["1"],
4309                "failed for {html}"
4310            );
4311        }
4312    }
4313
4314    #[test]
4315    fn an_empty_or_absent_label_is_not_a_reference() {
4316        for html in [
4317            r#"<p>a<sup data-footnote-ref=""></sup></p>"#,
4318            r#"<p>a<sup></sup></p>"#,
4319        ] {
4320            let blocks = parse_html(html);
4321            assert!(
4322                blocks
4323                    .iter()
4324                    .flat_map(|b| b.spans.iter())
4325                    .all(|s| s.footnote_ref.is_none()),
4326                "failed for {html}"
4327            );
4328        }
4329    }
4330}
4331
4332#[cfg(test)]
4333mod html_depth_tests {
4334    //! HTML nested more deeply than the reader follows, and HTML table cells.
4335
4336    use super::*;
4337
4338    /// The text of each block, in order.
4339    fn block_texts(blocks: &[ParsedBlock]) -> Vec<String> {
4340        blocks
4341            .iter()
4342            .map(|block| block.spans.iter().map(|span| span.text.as_str()).collect())
4343            .collect()
4344    }
4345
4346    /// An HTML list nested past the deepest level an insertion keeps comes in with every
4347    /// item, in its order, the deeper ones side by side at that level. The reader stopped
4348    /// descending at its recursion limit and dropped every item below it (the eighty-sixth
4349    /// level on).
4350    #[test]
4351    fn a_deep_html_list_keeps_every_item_at_the_deepest_level() {
4352        use crate::parser_tools::list_depth::MAX_LIST_INDENT;
4353        let depth = 300;
4354        let mut html: String = (0..depth).map(|i| format!("<ol><li>item {i}")).collect();
4355        html.push_str(&"</li></ol>".repeat(depth));
4356        let blocks = parse_html(&html);
4357        let items: Vec<(String, u32)> = blocks
4358            .iter()
4359            .filter(|block| block.list_style.is_some())
4360            .map(|block| {
4361                let text: String = block.spans.iter().map(|span| span.text.as_str()).collect();
4362                (text, block.list_indent)
4363            })
4364            .collect();
4365        let expected: Vec<(String, u32)> = (0..depth)
4366            .map(|i| (format!("item {i}"), (i as i64).min(MAX_LIST_INDENT) as u32))
4367            .collect();
4368        assert_eq!(items, expected);
4369    }
4370
4371    /// Text outside any item of a list nested too deeply stays, after the items.
4372    #[test]
4373    fn text_in_a_deep_html_list_outside_its_items_is_kept() {
4374        let depth = 40;
4375        let mut html: String = (0..depth).map(|i| format!("<ul><li>item {i}")).collect();
4376        html.push_str("<ul>loose words<li>last</li></ul>");
4377        html.push_str(&"</li></ul>".repeat(depth));
4378        let texts = block_texts(&parse_html(&html)).join(" ");
4379        assert!(texts.contains("loose words"), "{texts:?}");
4380        assert!(texts.contains("last"), "{texts:?}");
4381        assert!(texts.contains("item 39"), "{texts:?}");
4382    }
4383
4384    /// Past the depth the reader follows, it used to keep nothing: the words at the bottom
4385    /// of three hundred nested `<div>`s, or quotations, were dropped from the paste.
4386    #[test]
4387    fn html_nested_past_the_reader_depth_keeps_its_words() {
4388        for (open, close) in [("<div>", "</div>"), ("<blockquote>", "</blockquote>")] {
4389            let html = format!(
4390                "<p>Top.</p>{}<p>Deep <b>words</b></p><p>here.</p>{}<p>End.</p>",
4391                open.repeat(300),
4392                close.repeat(300)
4393            );
4394            let texts = block_texts(&parse_html(&html)).join(" ");
4395            let words: Vec<&str> = texts.split_whitespace().collect();
4396            assert_eq!(
4397                words,
4398                ["Top.", "Deep", "words", "here.", "End."],
4399                "for {open}"
4400            );
4401        }
4402    }
4403
4404    /// The same inside a table cell, read as one paragraph.
4405    #[test]
4406    fn a_table_cell_nested_past_the_reader_depth_keeps_its_words() {
4407        let html = format!(
4408            "<table><tr><td>{}deep words{}</td></tr></table>",
4409            "<span>".repeat(300),
4410            "</span>".repeat(300)
4411        );
4412        let elements = parse_html_elements(&html);
4413        let Some(ParsedElement::Table(table)) = elements.first() else {
4414            panic!("a table: {elements:?}");
4415        };
4416        let text: String = table.rows[0][0]
4417            .spans
4418            .iter()
4419            .map(|span| span.text.as_str())
4420            .collect();
4421        assert_eq!(text, "deep words");
4422    }
4423
4424    /// A cell is one paragraph, so its line breaks and paragraphs are spaces between
4425    /// words. The HTML writer puts a cell's two paragraphs out as `One<br/>Two`, and the
4426    /// reader read them back as the one word `OneTwo`.
4427    #[test]
4428    fn line_breaks_and_paragraphs_in_an_html_cell_separate_words() {
4429        for cell in [
4430            "One<br/>Two",
4431            "<p>One</p><p>Two</p>",
4432            "<ul><li>One</li><li>Two</li></ul>",
4433        ] {
4434            let elements = parse_html_elements(&format!("<table><tr><td>{cell}</td></tr></table>"));
4435            let Some(ParsedElement::Table(table)) = elements.first() else {
4436                panic!("a table: {elements:?}");
4437            };
4438            let text: String = table.rows[0][0]
4439                .spans
4440                .iter()
4441                .map(|span| span.text.as_str())
4442                .collect();
4443            assert_eq!(text, "One Two", "for {cell}");
4444        }
4445    }
4446}