Skip to main content

text_document/
fragment.rs

1//! DocumentFragment — format-agnostic rich text interchange type.
2
3use crate::{CharVerticalAlignment, InlineContent, LINE_SEPARATOR, ListStyle};
4use frontend::common::parser_tools::content_parser::{
5    ParsedBlock, ParsedElement, ParsedSpan, html_line_ends_before, html_text_ends_open,
6    push_html_text, split_block_at_line_breaks, task_box_markdown, task_checkbox_end_html,
7    task_checkbox_html,
8};
9use frontend::common::parser_tools::fragment_schema::{
10    FragmentBlock, FragmentData, FragmentElement, FragmentListStarts, FragmentQuoting,
11    FragmentTable, FragmentTableCell, as_a_whole_text, fragment_from_json,
12    fragment_with_list_starts_to_json,
13};
14
15/// A piece of rich text that can be inserted into a [`TextDocument`](crate::TextDocument).
16///
17/// `DocumentFragment` is the clipboard/interchange type. It carries
18/// blocks, per-character format runs, image anchors, and structural
19/// metadata in a format-agnostic internal representation.
20#[derive(Debug, Clone)]
21pub struct DocumentFragment {
22    data: String,
23    plain_text: String,
24}
25
26impl DocumentFragment {
27    /// Create an empty fragment.
28    pub fn new() -> Self {
29        Self {
30            data: String::new(),
31            plain_text: String::new(),
32        }
33    }
34
35    /// Create a fragment from plain text.
36    ///
37    /// Builds valid fragment data so the fragment can be inserted via
38    /// [`TextCursor::insert_fragment`](crate::TextCursor::insert_fragment).
39    pub fn from_plain_text(text: &str) -> Self {
40        let blocks: Vec<FragmentBlock> = text
41            .split('\n')
42            .map(|line| FragmentBlock {
43                plain_text: line.to_string(),
44                elements: vec![FragmentElement {
45                    content: InlineContent::Text(line.to_string()),
46                    fmt_font_family: None,
47                    fmt_font_point_size: None,
48                    fmt_font_weight: None,
49                    fmt_font_bold: None,
50                    fmt_font_italic: None,
51                    fmt_font_underline: None,
52                    fmt_font_overline: None,
53                    fmt_font_strikeout: None,
54                    fmt_letter_spacing: None,
55                    fmt_word_spacing: None,
56                    fmt_anchor_href: None,
57                    fmt_anchor_names: vec![],
58                    fmt_is_anchor: None,
59                    fmt_tooltip: None,
60                    fmt_underline_style: None,
61                    fmt_vertical_alignment: None,
62                }],
63                heading_level: None,
64                list: None,
65                alignment: None,
66                indent: None,
67                text_indent: None,
68                marker: None,
69                top_margin: None,
70                bottom_margin: None,
71                left_margin: None,
72                right_margin: None,
73                tab_positions: vec![],
74                line_height: None,
75                non_breakable_lines: None,
76                page_break_before: None,
77                direction: None,
78                background_color: None,
79                is_code_block: None,
80                code_language: None,
81                hyphenate: None,
82                language: None,
83            })
84            .collect();
85
86        let data = serde_json::to_string(&FragmentData {
87            blocks,
88            tables: vec![],
89        })
90        .expect("fragment serialization should not fail");
91
92        Self {
93            data,
94            plain_text: text.to_string(),
95        }
96    }
97
98    /// Create a fragment from HTML, as a clipboard carries a passage copied from a web page
99    /// or another application.
100    ///
101    /// A `<pre>` goes in as paragraphs, one for each of its lines, each keeping its spaces,
102    /// where [`TextDocument::set_html`](crate::TextDocument::set_html) loads it as a code
103    /// block: a preformatted passage pasted into prose is most often a poem or a letter set
104    /// line by line, and a code block keeps no emphasis and takes no footnote reference.
105    pub fn from_html(html: &str) -> Self {
106        let parsed = frontend::common::parser_tools::content_parser::parse_html_elements(html);
107        parsed_elements_to_fragment(parsed, ParsedFrom::Html)
108    }
109
110    /// Create a fragment from Markdown. The fragment is a whole text: see
111    /// [`from_djot`](Self::from_djot).
112    pub fn from_markdown(markdown: &str) -> Self {
113        let parsed = frontend::common::parser_tools::content_parser::parse_markdown(markdown);
114        parsed_elements_to_fragment(parsed, ParsedFrom::SavedSyntax)
115    }
116
117    /// Create a fragment from djot markup. Paste always uses the lossless
118    /// default [`crate::DjotImportOptions`]; per-feature selection is exposed on
119    /// the document-level import path (`TextDocument::set_djot_with_options`).
120    ///
121    /// The fragment is a whole text: inserted over a selection holding the whole of a
122    /// text, it reads as the same Djot loaded, even as a single plain paragraph (see
123    /// [`TextCursor::insert_fragment`](crate::TextCursor::insert_fragment)).
124    pub fn from_djot(djot: &str) -> Self {
125        let parsed = frontend::common::parser_tools::content_parser::parse_djot(
126            djot,
127            &frontend::common::parser_tools::DjotImportOptions::default(),
128        );
129        parsed_elements_to_fragment(parsed, ParsedFrom::SavedSyntax)
130    }
131
132    /// Create a fragment from an entire document. The fragment is a whole text: see
133    /// [`from_djot`](Self::from_djot).
134    pub fn from_document(doc: &crate::TextDocument) -> crate::Result<Self> {
135        let inner = doc.inner.lock();
136        // Use i64::MAX as anchor to ensure the full document is captured.
137        // Document positions include inter-block gaps, so character_count
138        // alone would truncate the last block.
139        let dto = frontend::document_inspection::ExtractFragmentDto {
140            position: 0,
141            anchor: i64::MAX,
142        };
143        let result =
144            frontend::commands::document_inspection_commands::extract_fragment(&inner.ctx, &dto)?;
145        Ok(Self::from_raw(
146            as_a_whole_text(&result.fragment_data),
147            result.plain_text,
148        ))
149    }
150
151    /// Create a fragment from the serialized internal format.
152    pub(crate) fn from_raw(data: String, plain_text: String) -> Self {
153        Self { data, plain_text }
154    }
155
156    /// Export the fragment as plain text.
157    pub fn to_plain_text(&self) -> &str {
158        &self.plain_text
159    }
160
161    /// Export the fragment as HTML.
162    pub fn to_html(&self) -> String {
163        if self.data.is_empty() {
164            return String::from("<html><head><meta charset=\"utf-8\"></head><body></body></html>");
165        }
166
167        let (fragment_data, list_starts) = match fragment_from_json(&self.data) {
168            Ok(carried) => (carried.data, carried.list_starts),
169            Err(_) => {
170                return String::from(
171                    "<html><head><meta charset=\"utf-8\"></head><body></body></html>",
172                );
173            }
174        };
175
176        let mut body = String::new();
177        let blocks = &fragment_data.blocks;
178
179        // Single inline-only block with no tables: emit inline HTML without block wrapper
180        if blocks.len() == 1 && blocks[0].is_inline_only() && fragment_data.tables.is_empty() {
181            push_inline_html(&mut body, &blocks[0].elements);
182            return format!(
183                "<html><head><meta charset=\"utf-8\"></head><body>{}</body></html>",
184                body
185            );
186        }
187
188        // Sort tables by block_insert_index so we can interleave them
189        let mut sorted_tables: Vec<&FragmentTable> = fragment_data.tables.iter().collect();
190        sorted_tables.sort_by_key(|t| t.block_insert_index);
191        let mut table_cursor = 0;
192
193        let mut i = 0;
194
195        while i < blocks.len() {
196            // Insert any tables whose block_insert_index == i
197            while table_cursor < sorted_tables.len()
198                && sorted_tables[table_cursor].block_insert_index <= i
199            {
200                push_table_html(&mut body, sorted_tables[table_cursor]);
201                table_cursor += 1;
202            }
203
204            let block = &blocks[i];
205
206            if let Some(ref list) = block.list {
207                let is_ordered = is_ordered_list_style(&list.style);
208                let list_tag = if is_ordered { "ol" } else { "ul" };
209                body.push('<');
210                body.push_str(list_tag);
211                // Where the list starts, when it is not 1.
212                if let Some(start) = list_starts.get(i).filter(|_| is_ordered) {
213                    body.push_str(&format!(" start=\"{start}\""));
214                }
215                body.push('>');
216
217                while i < blocks.len() {
218                    let b = &blocks[i];
219                    match &b.list {
220                        Some(l) if is_ordered_list_style(&l.style) == is_ordered => {
221                            body.push_str("<li>");
222                            body.push_str(task_checkbox_html(b.marker.as_ref()));
223                            push_inline_html(&mut body, &b.elements);
224                            body.push_str(task_checkbox_end_html(b.marker.as_ref()));
225                            body.push_str("</li>");
226                            i += 1;
227                        }
228                        _ => break,
229                    }
230                }
231
232                body.push_str("</");
233                body.push_str(list_tag);
234                body.push('>');
235            } else if let Some(level) = block.heading_level {
236                let n = level.clamp(1, 6);
237                body.push_str(&format!("<h{}>", n));
238                push_inline_html(&mut body, &block.elements);
239                body.push_str(&format!("</h{}>", n));
240                i += 1;
241            } else {
242                // Emit block-level formatting as inline styles (ISSUE-19)
243                let style = block_style_attr(block);
244                if style.is_empty() {
245                    body.push_str("<p>");
246                } else {
247                    body.push_str(&format!("<p style=\"{}\">", style));
248                }
249                push_inline_html(&mut body, &block.elements);
250                body.push_str("</p>");
251                i += 1;
252            }
253        }
254
255        // Emit any remaining tables after all blocks
256        while table_cursor < sorted_tables.len() {
257            push_table_html(&mut body, sorted_tables[table_cursor]);
258            table_cursor += 1;
259        }
260
261        format!(
262            "<html><head><meta charset=\"utf-8\"></head><body>{}</body></html>",
263            body
264        )
265    }
266
267    /// Export the fragment as Markdown.
268    pub fn to_markdown(&self) -> String {
269        if self.data.is_empty() {
270            return String::new();
271        }
272
273        let (fragment_data, list_starts) = match fragment_from_json(&self.data) {
274            Ok(carried) => (carried.data, carried.list_starts),
275            Err(_) => return String::new(),
276        };
277
278        // (rendered_text, is_list_item) — used for join logic
279        let mut parts: Vec<(String, bool)> = Vec::new();
280        let mut prev_was_list = false;
281        let mut list_counter: u32 = 0;
282
283        // Sort tables by block_insert_index for interleaving
284        let mut sorted_tables: Vec<&FragmentTable> = fragment_data.tables.iter().collect();
285        sorted_tables.sort_by_key(|t| t.block_insert_index);
286        let mut table_cursor = 0;
287
288        for (blk_idx, block) in fragment_data.blocks.iter().enumerate() {
289            // Insert tables before this block index
290            while table_cursor < sorted_tables.len()
291                && sorted_tables[table_cursor].block_insert_index <= blk_idx
292            {
293                parts.push((render_table_markdown(sorted_tables[table_cursor]), false));
294                prev_was_list = false;
295                list_counter = 0;
296                table_cursor += 1;
297            }
298
299            // An ATX heading is one line, so its line breaks are spaces.
300            let line_break = if block.heading_level.is_some() {
301                " "
302            } else {
303                "\\\n"
304            };
305            let inline_text = render_inline_markdown(&block.elements, line_break);
306            let is_list = block.list.is_some();
307
308            let indent_prefix = match block.indent {
309                Some(n) if n > 0 => "  ".repeat(n as usize),
310                _ => String::new(),
311            };
312
313            if let Some(level) = block.heading_level {
314                let n = level.clamp(1, 6) as usize;
315                let prefix = "#".repeat(n);
316                parts.push((format!("{} {}", prefix, inline_text), false));
317                prev_was_list = false;
318                list_counter = 0;
319            } else if let Some(ref list) = block.list {
320                let is_ordered = is_ordered_list_style(&list.style);
321                let task_box = task_box_markdown(block.marker.as_ref());
322                if !prev_was_list {
323                    // From where the list starts, when it is not 1.
324                    list_counter = list_starts
325                        .get(blk_idx)
326                        .and_then(|start| u32::try_from(start.saturating_sub(1)).ok())
327                        .unwrap_or(0);
328                }
329                if is_ordered {
330                    list_counter += 1;
331                    // `)` stays `)`; every other delimiter is written as `.`, the only
332                    // other one CommonMark has.
333                    let delimiter = if list.suffix == ")" { ')' } else { '.' };
334                    parts.push((
335                        format!(
336                            "{}{}{} {}{}",
337                            indent_prefix, list_counter, delimiter, task_box, inline_text
338                        ),
339                        true,
340                    ));
341                } else {
342                    parts.push((
343                        format!("{}- {}{}", indent_prefix, task_box, inline_text),
344                        true,
345                    ));
346                }
347                prev_was_list = true;
348            } else {
349                if indent_prefix.is_empty() {
350                    parts.push((inline_text, false));
351                } else {
352                    parts.push((format!("{}{}", indent_prefix, inline_text), false));
353                }
354                prev_was_list = false;
355                list_counter = 0;
356            }
357
358            if !is_list {
359                prev_was_list = false;
360            }
361        }
362
363        // Emit remaining tables after all blocks
364        while table_cursor < sorted_tables.len() {
365            parts.push((render_table_markdown(sorted_tables[table_cursor]), false));
366            table_cursor += 1;
367        }
368
369        // Join: list items with \n, others with \n\n
370        let mut result = String::new();
371        for (idx, (text, is_list)) in parts.iter().enumerate() {
372            if idx > 0 {
373                let (_, prev_is_list) = &parts[idx - 1];
374                if *prev_is_list && *is_list {
375                    result.push('\n');
376                } else {
377                    result.push_str("\n\n");
378                }
379            }
380            result.push_str(text);
381        }
382
383        result
384    }
385
386    /// Returns true if the fragment contains no text or elements.
387    pub fn is_empty(&self) -> bool {
388        self.plain_text.is_empty()
389    }
390
391    /// Returns the serialized internal representation.
392    pub(crate) fn raw_data(&self) -> &str {
393        &self.data
394    }
395}
396
397impl Default for DocumentFragment {
398    fn default() -> Self {
399        Self::new()
400    }
401}
402
403// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
404// Shared helpers (used by both to_html and to_markdown)
405// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
406
407fn is_ordered_list_style(style: &ListStyle) -> bool {
408    matches!(
409        style,
410        ListStyle::Decimal
411            | ListStyle::LowerAlpha
412            | ListStyle::UpperAlpha
413            | ListStyle::LowerRoman
414            | ListStyle::UpperRoman
415    )
416}
417
418// ── HTML helpers ────────────────────────────────────────────────
419
420fn escape_html(s: &str) -> String {
421    let mut out = String::with_capacity(s.len());
422    for c in s.chars() {
423        match c {
424            '&' => out.push_str("&amp;"),
425            '<' => out.push_str("&lt;"),
426            '>' => out.push_str("&gt;"),
427            '"' => out.push_str("&quot;"),
428            '\'' => out.push_str("&#x27;"),
429            // A raw CR in text content is normalised to LF by the HTML5 input
430            // preprocessor on re-import (CR-from-`&#xD;` survives, literal CR
431            // does not), which breaks serialiser idempotency. Emit it as a
432            // numeric reference so it round-trips losslessly.
433            '\r' => out.push_str("&#13;"),
434            _ => out.push(c),
435        }
436    }
437    out
438}
439
440/// Build a CSS `style` attribute value from block-level formatting (ISSUE-19).
441fn block_style_attr(block: &FragmentBlock) -> String {
442    use crate::Alignment;
443
444    let mut parts = Vec::new();
445    if let Some(ref alignment) = block.alignment {
446        let value = match alignment {
447            Alignment::Left => "left",
448            Alignment::Right => "right",
449            Alignment::Center => "center",
450            Alignment::Justify => "justify",
451        };
452        parts.push(format!("text-align: {}", value));
453    }
454    if let Some(n) = block.indent
455        && n > 0
456    {
457        parts.push(format!("margin-left: {}em", n));
458    }
459    if let Some(px) = block.text_indent
460        && px != 0
461    {
462        parts.push(format!("text-indent: {}px", px));
463    }
464    if let Some(px) = block.top_margin {
465        parts.push(format!("margin-top: {}px", px));
466    }
467    if let Some(px) = block.bottom_margin {
468        parts.push(format!("margin-bottom: {}px", px));
469    }
470    if let Some(px) = block.left_margin {
471        parts.push(format!("margin-left: {}px", px));
472    }
473    if let Some(px) = block.right_margin {
474        parts.push(format!("margin-right: {}px", px));
475    }
476    parts.join("; ")
477}
478
479fn push_inline_html(out: &mut String, elements: &[FragmentElement]) {
480    // Whether the text before an element, on its line, ends in whitespace a reader would
481    // collapse a space after it into (see `push_html_text`).
482    let mut after_space = true;
483    for (index, elem) in elements.iter().enumerate() {
484        let text = match &elem.content {
485            InlineContent::Text(t) => {
486                let ends_line =
487                    html_line_ends_before(elements[index + 1..].iter().map(|e| &e.content));
488                let mut html = String::new();
489                push_html_text(&mut html, t, after_space, ends_line, escape_html, "<br>");
490                after_space = html_text_ends_open(t, after_space);
491                html
492            }
493            // A reference carried onto the clipboard keeps its marker; the note
494            // body is not part of the fragment.
495            InlineContent::FootnoteRef { label } => {
496                after_space = false;
497                let id = escape_html(label);
498                out.push_str(&format!(
499                    "<a epub:type=\"noteref\" role=\"doc-noteref\" href=\"#fn-{id}\"><sup>{id}</sup></a>"
500                ));
501                continue;
502            }
503            InlineContent::Image {
504                name,
505                alt,
506                width,
507                height,
508                ..
509            } => {
510                after_space = false;
511                // This is the HTML written to the OS clipboard, so it is what
512                // another application receives on paste. It carried no `alt` at
513                // all, which made every copied image inaccessible in the target
514                // document.
515                let mut tag = format!(
516                    "<img src=\"{}\" alt=\"{}\"",
517                    escape_html(name),
518                    escape_html(alt)
519                );
520                if *width > 0 {
521                    tag.push_str(&format!(" width=\"{width}\""));
522                }
523                if *height > 0 {
524                    tag.push_str(&format!(" height=\"{height}\""));
525                }
526                tag.push('>');
527                tag
528            }
529            InlineContent::Empty => String::new(),
530        };
531
532        let is_monospace = elem
533            .fmt_font_family
534            .as_deref()
535            .is_some_and(|f| f == "monospace");
536        let is_bold = elem.fmt_font_bold.unwrap_or(false);
537        let is_italic = elem.fmt_font_italic.unwrap_or(false);
538        let is_underline = elem.fmt_font_underline.unwrap_or(false);
539        let is_strikeout = elem.fmt_font_strikeout.unwrap_or(false);
540        let is_anchor = elem.fmt_is_anchor.unwrap_or(false);
541        // Superscript and subscript are character formatting like the rest, and
542        // an exponent or a chemical index that lands on the baseline is simply
543        // the wrong text — so they belong in the clipboard payload alongside
544        // bold and italic rather than being dropped on the way out.
545        let vertical = elem.fmt_vertical_alignment.as_ref();
546
547        let mut result = text;
548
549        if is_monospace {
550            result = format!("<code>{}</code>", result);
551        }
552        if is_bold {
553            result = format!("<strong>{}</strong>", result);
554        }
555        if is_italic {
556            result = format!("<em>{}</em>", result);
557        }
558        if is_underline {
559            result = format!("<u>{}</u>", result);
560        }
561        if is_strikeout {
562            result = format!("<s>{}</s>", result);
563        }
564        match vertical {
565            Some(CharVerticalAlignment::SuperScript) => result = format!("<sup>{result}</sup>"),
566            Some(CharVerticalAlignment::SubScript) => result = format!("<sub>{result}</sub>"),
567            _ => {}
568        }
569        if is_anchor && let Some(ref href) = elem.fmt_anchor_href {
570            result = format!("<a href=\"{}\">{}</a>", escape_html(href), result);
571        }
572
573        out.push_str(&result);
574    }
575}
576
577/// Emit an HTML `<table>` for a `FragmentTable`.
578fn push_table_html(out: &mut String, table: &FragmentTable) {
579    out.push_str("<table>");
580    for row in 0..table.rows {
581        out.push_str("<tr>");
582        for col in 0..table.columns {
583            if let Some(cell) = table.cells.iter().find(|c| c.row == row && c.column == col) {
584                out.push_str("<td");
585                if cell.row_span > 1 {
586                    out.push_str(&format!(" rowspan=\"{}\"", cell.row_span));
587                }
588                if cell.column_span > 1 {
589                    out.push_str(&format!(" colspan=\"{}\"", cell.column_span));
590                }
591                out.push('>');
592                for (i, block) in cell.blocks.iter().enumerate() {
593                    if i > 0 {
594                        out.push_str("<br>");
595                    }
596                    push_inline_html(out, &block.elements);
597                }
598                out.push_str("</td>");
599            }
600            // Skip positions covered by spans — the HTML renderer handles them.
601        }
602        out.push_str("</tr>");
603    }
604    out.push_str("</table>");
605}
606
607// ── Markdown helpers ────────────────────────────────────────────
608
609fn escape_markdown(s: &str) -> String {
610    let mut out = String::with_capacity(s.len());
611    for c in s.chars() {
612        if matches!(
613            c,
614            '\\' | '`'
615                | '*'
616                | '_'
617                | '{'
618                | '}'
619                | '['
620                | ']'
621                | '('
622                | ')'
623                | '#'
624                | '+'
625                | '-'
626                | '.'
627                | '!'
628                | '|'
629                | '~'
630                | '<'
631                | '>'
632        ) {
633            out.push('\\');
634        }
635        out.push(c);
636    }
637    out
638}
639
640/// A block's inline content as Markdown, each line break inside it
641/// ([`LINE_SEPARATOR`]) written as `line_break`: a backslash hard break in a block, a
642/// space in a heading or a table cell, which are one line.
643///
644/// A break at the very end of the block is left out: a backslash there is not a hard
645/// break to a reader but the text `\`, and Markdown has nothing else to write it as. A
646/// code span cannot hold a hard break, so one over a break is written as one code span a
647/// line.
648fn render_inline_markdown(elements: &[FragmentElement], line_break: &str) -> String {
649    // The breaks the block ends with, in whichever elements hold them.
650    let mut trailing = 0usize;
651    for elem in elements.iter().rev() {
652        match &elem.content {
653            InlineContent::Text(t) => {
654                let kept = t.trim_end_matches(LINE_SEPARATOR);
655                trailing += t.len() - kept.len();
656                if !kept.is_empty() {
657                    break;
658                }
659            }
660            InlineContent::Empty => {}
661            _ => break,
662        }
663    }
664    let mut to_drop = trailing;
665    let trimmed: Vec<FragmentElement> = elements
666        .iter()
667        .rev()
668        .map(|elem| match &elem.content {
669            InlineContent::Text(t) if to_drop > 0 => {
670                let keep = t.len().saturating_sub(to_drop);
671                to_drop -= t.len() - keep;
672                FragmentElement {
673                    content: InlineContent::Text(t[..keep].to_string()),
674                    ..elem.clone()
675                }
676            }
677            _ => elem.clone(),
678        })
679        .collect::<Vec<_>>()
680        .into_iter()
681        .rev()
682        .collect();
683    let mut out = String::new();
684    for elem in &trimmed {
685        let raw_text = match &elem.content {
686            InlineContent::Text(t) => t.clone(),
687            // `name` was used as both alt and source, so a pasted image
688            // described itself with its filename.
689            InlineContent::Image { name, alt, .. } => format!("![{alt}]({name})"),
690            InlineContent::FootnoteRef { label } => format!("[^{label}]"),
691            InlineContent::Empty => String::new(),
692        };
693
694        let is_monospace = elem
695            .fmt_font_family
696            .as_deref()
697            .is_some_and(|f| f == "monospace");
698        let is_bold = elem.fmt_font_bold.unwrap_or(false);
699        let is_italic = elem.fmt_font_italic.unwrap_or(false);
700        let is_strikeout = elem.fmt_font_strikeout.unwrap_or(false);
701        let is_anchor = elem.fmt_is_anchor.unwrap_or(false);
702
703        if is_monospace {
704            // A code span is one line: one a line, the break between them.
705            let spans: Vec<String> = raw_text
706                .split(LINE_SEPARATOR)
707                .map(|line| {
708                    if line.is_empty() {
709                        String::new()
710                    } else {
711                        format!("`{line}`")
712                    }
713                })
714                .collect();
715            out.push_str(&spans.join(line_break));
716        } else {
717            let mut text = escape_markdown(&raw_text).replace(LINE_SEPARATOR, line_break);
718            if is_bold && is_italic {
719                text = format!("***{}***", text);
720            } else if is_bold {
721                text = format!("**{}**", text);
722            } else if is_italic {
723                text = format!("*{}*", text);
724            }
725            if is_strikeout {
726                text = format!("~~{}~~", text);
727            }
728            if is_anchor {
729                let href = elem.fmt_anchor_href.as_deref().unwrap_or("");
730                out.push_str(&format!("[{}]({})", text, href));
731            } else {
732                out.push_str(&text);
733            }
734        }
735    }
736    out
737}
738
739/// Render a `FragmentTable` as a pipe-delimited Markdown table.
740fn render_table_markdown(table: &FragmentTable) -> String {
741    let mut rows: Vec<Vec<String>> = vec![vec![String::new(); table.columns]; table.rows];
742
743    for cell in &table.cells {
744        let text: String = cell
745            .blocks
746            .iter()
747            .map(|b| render_inline_markdown(&b.elements, " "))
748            .collect::<Vec<_>>()
749            .join(" ");
750        if cell.row < table.rows && cell.column < table.columns {
751            rows[cell.row][cell.column] = text;
752        }
753    }
754
755    let mut out = String::new();
756    for (i, row) in rows.iter().enumerate() {
757        out.push_str("| ");
758        out.push_str(&row.join(" | "));
759        out.push_str(" |");
760        if i == 0 {
761            // Header separator
762            out.push('\n');
763            out.push('|');
764            for _ in 0..table.columns {
765                out.push_str(" --- |");
766            }
767        }
768        if i + 1 < rows.len() {
769            out.push('\n');
770        }
771    }
772    out
773}
774
775// ── Fragment construction from parsed content ───────────────────
776
777/// Convert parsed blocks (from HTML or Markdown parser) into a `DocumentFragment`.
778/// Convert a `ParsedSpan` to a `FragmentElement`.
779/// The plain text of a run of parsed spans, with an image standing as its
780/// `U+FFFC` sentinel.
781///
782/// That sentinel is the convention the *extract* side already writes (see
783/// `extract_fragment_uc`), and it is what `insert_fragment_uc` positions each
784/// `ImageAnchor` against — an image with no character in the text has nothing
785/// to anchor to, and every format run after it lands three bytes early.
786fn spans_plain_text(spans: &[ParsedSpan]) -> String {
787    spans
788        .iter()
789        .map(|s| {
790            // Every inline object contributes its sentinel: the anchors this
791            // fragment carries are positioned against *this* string, so an
792            // object missing from it puts every anchor after it out by three
793            // bytes.
794            if s.image.is_some() || s.footnote_ref.is_some() {
795                "\u{FFFC}"
796            } else {
797                s.text.as_str()
798            }
799        })
800        .collect()
801}
802
803fn span_to_fragment_element(span: &ParsedSpan) -> FragmentElement {
804    // An image span carries no text — the picture *is* the content. Emitting
805    // `Text("")` for it, which is what this did, produced a fragment with an
806    // empty element and no image anywhere: `insert_djot` of an image markup
807    // returned `Ok` and inserted nothing at all.
808    //
809    // A footnote reference is the same shape and the same trap: its span carries
810    // no text either, so the identical `Text("")` would have made
811    // `insert_footnote_reference` a no-op that reported success.
812    let content = match (&span.footnote_ref, &span.image) {
813        (Some(label), _) => InlineContent::FootnoteRef {
814            label: label.clone(),
815        },
816        (None, Some(img)) => InlineContent::Image {
817            name: img.src.clone(),
818            alt: img.alt.clone(),
819            width: img.width,
820            height: img.height,
821            // The source states a display size or it does not; nothing here
822            // re-encodes the image, so there is no quality to carry.
823            quality: 100,
824        },
825        (None, None) => InlineContent::Text(span.text.clone()),
826    };
827    let fmt_font_family = if span.code {
828        Some("monospace".into())
829    } else {
830        None
831    };
832    let fmt_font_bold = if span.bold { Some(true) } else { None };
833    let fmt_font_italic = if span.italic { Some(true) } else { None };
834    let fmt_font_underline = if span.underline { Some(true) } else { None };
835    let fmt_font_strikeout = if span.strikeout { Some(true) } else { None };
836    let (fmt_anchor_href, fmt_is_anchor) = if let Some(ref href) = span.link_href {
837        (Some(href.clone()), Some(true))
838    } else {
839        (None, None)
840    };
841    // Raised and lowered text is character formatting like bold and italic, and
842    // dropping it here is what silently flattened an exponent or a chemical
843    // index on the way in: the parsers set the flags, `character_format_from_span`
844    // maps them, and only this conversion — the one every `insert_html` /
845    // `insert_markdown` / `insert_djot` goes through — threw them away.
846    let fmt_vertical_alignment = if span.superscript {
847        Some(CharVerticalAlignment::SuperScript)
848    } else if span.subscript {
849        Some(CharVerticalAlignment::SubScript)
850    } else {
851        None
852    };
853
854    FragmentElement {
855        content,
856        fmt_font_family,
857        fmt_font_point_size: None,
858        fmt_font_weight: None,
859        fmt_font_bold,
860        fmt_font_italic,
861        fmt_font_underline,
862        fmt_font_overline: None,
863        fmt_font_strikeout,
864        fmt_letter_spacing: None,
865        fmt_word_spacing: None,
866        fmt_anchor_href,
867        fmt_anchor_names: vec![],
868        fmt_is_anchor,
869        fmt_tooltip: None,
870        fmt_underline_style: None,
871        fmt_vertical_alignment,
872    }
873}
874
875/// A parsed block as a fragment block: its text and spans, and everything a load of the
876/// same text keeps of the block, so a text put back by selecting all and inserting it reads
877/// as the text loaded: its alignment, marker, spacing and list markers, and its code-block
878/// format when `keep_code_block` says so. Only the heading and the line attributes were
879/// carried, and every code block of a restored version came back as a plain paragraph. The
880/// quotations the block stands in travel beside it (see [`FragmentQuoting`]).
881fn parsed_block_to_fragment_block(pb: ParsedBlock, keep_code_block: bool) -> FragmentBlock {
882    use frontend::common::parser_tools::fragment_schema::FragmentList;
883
884    let elements: Vec<FragmentElement> = pb.spans.iter().map(span_to_fragment_element).collect();
885    let plain_text: String = spans_plain_text(&pb.spans);
886    let list = pb.list_style.map(|style| FragmentList {
887        style,
888        indent: pb.list_indent as i64,
889        prefix: pb.list_prefix,
890        suffix: pb.list_suffix,
891    });
892    let is_code_block = keep_code_block && pb.is_code_block;
893    FragmentBlock {
894        plain_text,
895        elements,
896        heading_level: pb.heading_level,
897        list,
898        alignment: pb.alignment,
899        indent: None,
900        text_indent: pb.text_indent,
901        marker: pb.marker,
902        top_margin: pb.top_margin,
903        bottom_margin: None,
904        left_margin: None,
905        right_margin: None,
906        tab_positions: vec![],
907        line_height: pb.line_height,
908        non_breakable_lines: pb.non_breakable_lines,
909        page_break_before: pb.page_break_before,
910        direction: pb.direction,
911        background_color: pb.background_color,
912        is_code_block: is_code_block.then_some(true),
913        code_language: if is_code_block {
914            pb.code_language
915        } else {
916            None
917        },
918        hyphenate: None,
919        language: None,
920    }
921}
922
923/// What a fragment was parsed from, which decides what an insertion makes of it.
924#[derive(Debug, Clone, Copy, PartialEq, Eq)]
925enum ParsedFrom {
926    /// A text in a syntax a text is saved in, Djot or Markdown: a whole text, whose code
927    /// blocks are code blocks. Inserted over a selection holding the whole of a text, it
928    /// reads as the same text loaded (see `whole_text_to_json`).
929    SavedSyntax,
930    /// HTML, as a clipboard carries a passage from a web page or another application. A
931    /// `<pre>` is read as paragraphs, one for each of its lines: a code block keeps no
932    /// emphasis, takes no footnote reference, and a host with no command to make it prose
933    /// again left the writer with a passage set as code.
934    Html,
935}
936
937/// Convert parsed elements (blocks + tables) into a `DocumentFragment`,
938/// preserving table structure as `FragmentTable` entries.
939fn parsed_elements_to_fragment(parsed: Vec<ParsedElement>, from: ParsedFrom) -> DocumentFragment {
940    let mut blocks: Vec<FragmentBlock> = Vec::new();
941    let mut tables: Vec<FragmentTable> = Vec::new();
942    // The quotations each block and table stands in, carried beside them.
943    let mut quoting = FragmentQuoting::default();
944    // The number each list starts at, carried beside its items: every item of a list
945    // starting past 1 names it, and an insertion reads it where it makes the list.
946    let mut list_starts = FragmentListStarts::default();
947
948    for elem in parsed {
949        match elem {
950            // A clipboard fragment carries prose, not note bodies: a definition
951            // has no position in the flow being copied, and pasting one would
952            // splice a note's text into the middle of a sentence. The reference
953            // travels; the body stays where it is defined.
954            ParsedElement::FootnoteDefinition { .. } => {}
955            // A code block of a text in a saved syntax is a code block, and holds its lines.
956            // Any other block is prose, and a `\n` in prose is a new block: one line a
957            // block, as a plain-text paste gives. A line break inside the block (a
958            // `LINE_SEPARATOR`) is not one of those, and stays. A pasted `<pre>` is prose too (see
959            // `ParsedFrom::Html`). Kept in one block, it held line breaks, which the Djot
960            // writer used to write as they were, and its lines read back as whatever their
961            // markers spelled.
962            ParsedElement::Block(pb) => {
963                let depth = pb.blockquote_depth;
964                // The quotations opening at the block open at its first line.
965                let mut opens = pb.blockquote_opens;
966                // The quotation's role rides on its first block, as the parsers put it.
967                let mut role = pb.semantic_role.clone();
968                let keep_code_block = pb.is_code_block && from == ParsedFrom::SavedSyntax;
969                let lines = if keep_code_block {
970                    vec![pb]
971                } else {
972                    split_block_at_line_breaks(pb)
973                };
974                for pb in lines {
975                    if let Some(start) = pb.list_start.filter(|_| pb.list_style.is_some()) {
976                        list_starts.set(blocks.len(), start);
977                    }
978                    blocks.push(parsed_block_to_fragment_block(pb, keep_code_block));
979                    if depth > 0 {
980                        quoting.set_block(blocks.len() - 1, depth, role.take());
981                        quoting.set_block_opens(blocks.len() - 1, std::mem::take(&mut opens));
982                    }
983                }
984            }
985            ParsedElement::Table(pt) => {
986                let block_insert_index = blocks.len();
987                let num_columns = pt.column_count();
988                let num_rows = pt.rows.len();
989
990                let mut frag_cells: Vec<FragmentTableCell> = Vec::new();
991                for (row_idx, row) in pt.rows.iter().enumerate() {
992                    // A row shorter than the widest is completed with empty cells, as a
993                    // load completes it: a table with holes in its grid is one no edit
994                    // expects.
995                    for col_idx in 0..num_columns {
996                        let spans = row
997                            .get(col_idx)
998                            .map_or(&[][..], |cell| cell.spans.as_slice());
999                        let cell_elements: Vec<FragmentElement> =
1000                            spans.iter().map(span_to_fragment_element).collect();
1001                        let cell_text: String = spans_plain_text(spans);
1002
1003                        frag_cells.push(FragmentTableCell {
1004                            row: row_idx,
1005                            column: col_idx,
1006                            row_span: 1,
1007                            column_span: 1,
1008                            blocks: vec![FragmentBlock {
1009                                plain_text: cell_text,
1010                                elements: cell_elements,
1011                                heading_level: None,
1012                                list: None,
1013                                alignment: None,
1014                                indent: None,
1015                                text_indent: None,
1016                                marker: None,
1017                                top_margin: None,
1018                                bottom_margin: None,
1019                                left_margin: None,
1020                                right_margin: None,
1021                                tab_positions: vec![],
1022                                line_height: None,
1023                                non_breakable_lines: None,
1024                                page_break_before: None,
1025                                direction: None,
1026                                background_color: None,
1027                                is_code_block: None,
1028                                code_language: None,
1029                                hyphenate: None,
1030                                language: None,
1031                            }],
1032                            fmt_padding: None,
1033                            fmt_border: None,
1034                            fmt_vertical_alignment: None,
1035                            fmt_background_color: None,
1036                        });
1037                    }
1038                }
1039
1040                tables.push(FragmentTable {
1041                    rows: num_rows,
1042                    columns: num_columns,
1043                    cells: frag_cells,
1044                    block_insert_index,
1045                    fmt_border: None,
1046                    fmt_cell_spacing: None,
1047                    fmt_cell_padding: None,
1048                    fmt_width: None,
1049                    fmt_alignment: None,
1050                    column_widths: vec![],
1051                });
1052                quoting.set_table(tables.len() - 1, pt.blockquote_depth);
1053                quoting.set_table_opens(tables.len() - 1, pt.blockquote_opens);
1054            }
1055        }
1056    }
1057
1058    // The blocks' own text, read before they are serialized: reading it back out of the
1059    // JSON parsed the whole fragment a second time.
1060    let plain_text = blocks
1061        .iter()
1062        .map(|b| b.plain_text.as_str())
1063        .collect::<Vec<_>>()
1064        .join("\n");
1065    let data = FragmentData { blocks, tables };
1066    let data = fragment_with_list_starts_to_json(
1067        &data,
1068        &quoting,
1069        &list_starts,
1070        from == ParsedFrom::SavedSyntax,
1071    )
1072    .expect("fragment serialization should not fail");
1073
1074    DocumentFragment { data, plain_text }
1075}
1076
1077#[cfg(test)]
1078mod tests {
1079    use super::*;
1080
1081    /// A paragraph as a manuscript holds it: emphasis, a link, a note reference.
1082    fn paragraph(i: usize) -> String {
1083        format!(
1084            "Paragraph {i} with *emphasis* and a [link](https://example.org/{i}) in it, and a \
1085             note[^n{i}] after the words that run on to a full line of prose."
1086        )
1087    }
1088
1089    /// A fragment is the text a cursor inserts, carried as JSON, and an insertion refuses
1090    /// JSON past 64 MiB. Every field of every block and every run was written, `null` or
1091    /// empty as most are: the JSON came to about 18 times the Djot it was made from, and
1092    /// putting back a whole text of 26,000 paragraphs failed. Fields holding nothing are
1093    /// left out now.
1094    #[test]
1095    fn a_fragment_holds_little_more_than_its_text() {
1096        let djot: String = (0..200).map(|i| format!("{}\n\n", paragraph(i))).collect();
1097        let fragment = DocumentFragment::from_djot(&djot);
1098        let ratio = fragment.raw_data().len() as f64 / djot.len() as f64;
1099        println!("fragment JSON: {ratio:.2} times the Djot");
1100        assert!(
1101            ratio < 4.0,
1102            "the fragment's JSON is {ratio:.1} times the Djot it was made from"
1103        );
1104        assert_eq!(fragment.to_plain_text().lines().count(), 200);
1105    }
1106
1107    /// A fragment written in full, as the builds before the compact form wrote it, still
1108    /// reads: every field left out reads as holding nothing.
1109    #[test]
1110    fn a_fragment_written_in_full_still_reads() {
1111        let full = r#"{"blocks":[{"plain_text":"Old","elements":[{"content":{"Text":"Old"},
1112            "fmt_font_family":null,"fmt_font_point_size":null,"fmt_font_weight":null,
1113            "fmt_font_bold":true,"fmt_font_italic":null,"fmt_font_underline":null,
1114            "fmt_font_overline":null,"fmt_font_strikeout":null,"fmt_letter_spacing":null,
1115            "fmt_word_spacing":null,"fmt_anchor_href":null,"fmt_anchor_names":[],
1116            "fmt_is_anchor":null,"fmt_tooltip":null,"fmt_underline_style":null,
1117            "fmt_vertical_alignment":null}],"heading_level":2,"list":null,"alignment":null,
1118            "indent":null,"text_indent":null,"marker":null,"top_margin":null,
1119            "bottom_margin":null,"left_margin":null,"right_margin":null,"tab_positions":[],
1120            "line_height":null,"non_breakable_lines":null,"direction":null,
1121            "background_color":null,"is_code_block":null,"code_language":null}]}"#;
1122        let data: FragmentData = serde_json::from_str(full).unwrap();
1123        assert_eq!(data.blocks[0].heading_level, Some(2));
1124        assert_eq!(data.blocks[0].elements[0].fmt_font_bold, Some(true));
1125        let compact = serde_json::to_string(&data).unwrap();
1126        let again: FragmentData = serde_json::from_str(&compact).unwrap();
1127        assert_eq!(again.blocks[0].heading_level, Some(2));
1128        assert_eq!(again.blocks[0].elements[0].fmt_font_bold, Some(true));
1129        assert!(compact.len() < full.len() / 3, "{compact}");
1130    }
1131}