Skip to main content

carve/
bbcode_migrate.rs

1//! BBCode-to-Carve migration.
2//!
3//! BBCode is a tag language, so tags are protected while ordinary text is
4//! escaped with the shared `plain` profile, then rewritten in a deliberate
5//! order. Literal/code runs are stashed until every other rewrite is complete.
6
7use crate::djot_migrate::{escape_plain_carve_syntax, HandledDelimiters};
8use regex::{Captures, Regex};
9use std::fmt;
10
11/// Maximum BBCode input length in bytes.
12pub const BBCODE_MAX_INPUT_LENGTH: usize = 262_144;
13
14/// Reason a BBCode import could not complete.
15#[derive(Debug, Clone, PartialEq, Eq)]
16pub enum BbcodeImportError {
17    /// Input exceeds the supported byte limit.
18    InputTooLarge {
19        /// Supplied input length in bytes.
20        bytes: usize,
21        /// Maximum allowed length in bytes.
22        maximum: usize,
23    },
24    /// No unused private-use sentinel remained for protecting literal content.
25    SentinelSpaceExhausted,
26}
27
28impl fmt::Display for BbcodeImportError {
29    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
30        match self {
31            Self::InputTooLarge { bytes, maximum } => write!(
32                f,
33                "BBCode input exceeds maximum length of {maximum} bytes (got {bytes})"
34            ),
35            Self::SentinelSpaceExhausted => f.write_str(
36                "BBCode input occupies every private-use sentinel available to the importer",
37            ),
38        }
39    }
40}
41
42impl std::error::Error for BbcodeImportError {}
43
44fn re(pattern: &str) -> Regex {
45    Regex::new(pattern).expect("the BBCode importer owns valid regexes")
46}
47
48fn replace(text: String, pattern: &str, replacement: &str) -> String {
49    re(pattern).replace_all(&text, replacement).into_owned()
50}
51
52fn pick_pair(source: &str, first: u32) -> Result<(char, char), BbcodeImportError> {
53    for code in first..=0xf8fd {
54        let Some(open) = char::from_u32(code) else {
55            continue;
56        };
57        let Some(close) = char::from_u32(code + 1) else {
58            continue;
59        };
60        if !source.contains(open) && !source.contains(close) {
61            return Ok((open, close));
62        }
63    }
64    Err(BbcodeImportError::SentinelSpaceExhausted)
65}
66
67fn pick_mark(source: &str, first: u32) -> Result<char, BbcodeImportError> {
68    let occupied: std::collections::HashSet<char> = source
69        .chars()
70        .filter(|c| ('\u{e000}'..='\u{f8ff}').contains(c))
71        .collect();
72    (first..=0xf8ff)
73        .chain(0xe000..first)
74        .filter_map(char::from_u32)
75        .find(|code| !occupied.contains(code))
76        .ok_or(BbcodeImportError::SentinelSpaceExhausted)
77}
78
79fn stash_matches(
80    mut text: String,
81    patterns: &[&str],
82    open: char,
83    close: char,
84    stash: &mut Vec<String>,
85) -> String {
86    for pattern in patterns {
87        text = re(pattern)
88            .replace_all(&text, |caps: &Captures<'_>| {
89                let index = stash.len();
90                stash.push(caps[0].to_string());
91                format!("{open}{index}{close}")
92            })
93            .into_owned();
94    }
95    text
96}
97
98fn restore(mut text: String, open: char, close: char, stash: &[String]) -> String {
99    let pattern = re(&format!(
100        r"{}([0-9]+){}",
101        regex::escape(&open.to_string()),
102        regex::escape(&close.to_string())
103    ));
104    loop {
105        let next = pattern
106            .replace_all(&text, |caps: &Captures<'_>| {
107                caps[1]
108                    .parse::<usize>()
109                    .ok()
110                    .and_then(|index| stash.get(index))
111                    .cloned()
112                    .unwrap_or_default()
113            })
114            .into_owned();
115        if next == text {
116            return text;
117        }
118        text = next;
119    }
120}
121
122fn escape_text(source: &str) -> Result<String, BbcodeImportError> {
123    let (open, close) = pick_pair(source, 0xe001)?;
124    let mut stash = Vec::new();
125    let mut text = stash_matches(
126        source.to_string(),
127        &[
128            r"(?is)\[code(?:=[^\]]*)?\].*?\[/code\]",
129            r"(?is)\[(?:c|icode)\].*?\[/(?:c|icode)\]",
130            r"(?is)\[url\].*?\[/url\]",
131            r"(?is)\[img(?:=[^\]]*)?\].*?\[/img\]",
132            // A quote opener with its fields after a space is a tag too.
133            r"(?i)\[quote [^\]]*\]",
134            r"(?i)\[/?[a-z][a-z0-9]*(?:=[^\]]*)?\]",
135            r"\[\*\]",
136        ],
137        open,
138        close,
139        &mut stash,
140    );
141    text = text.replace('\\', "\\\\").replace('`', "\\`");
142    text = text.replace("{#", "\\{#").replace("{.", "\\{.");
143    text = escape_plain_carve_syntax(&text, HandledDelimiters::PLAIN);
144    Ok(restore(text, open, close, &stash))
145}
146
147fn stash_literals(source: String) -> Result<(String, char, char, Vec<String>), BbcodeImportError> {
148    let (open, close) = pick_pair(&source, 0xe010)?;
149    let mut stash = Vec::new();
150    let mut text = source;
151    for (kind, pattern) in [
152        r"(?is)(\[code(?:=[^\]]*)?\])(.*?)(\[/code\])",
153        r"(?is)(\[(?:c|icode)\])(.*?)(\[/(?:c|icode)\])",
154    ]
155    .into_iter()
156    .enumerate()
157    {
158        text = re(pattern)
159            .replace_all(&text, |caps: &Captures<'_>| {
160                let index = stash.len();
161                stash.push(if kind == 0 {
162                    caps[2].trim().to_string()
163                } else {
164                    caps[2].to_string()
165                });
166                format!("{}{open}{index}{close}{}", &caps[1], &caps[3])
167            })
168            .into_owned();
169    }
170    text = re(r"(?is)\[noparse\](.*?)\[/noparse\]")
171        .replace_all(&text, |caps: &Captures<'_>| {
172            let index = stash.len();
173            stash.push(caps[1].to_string());
174            format!("{open}{index}{close}")
175        })
176        .into_owned();
177    Ok((text, open, close, stash))
178}
179
180fn convert_pairs(
181    mut text: String,
182    link_mark: Option<char>,
183    styling_drops: &mut Vec<(String, String)>,
184) -> String {
185    // The link and image passes mark each link they write, so the formatting
186    // pass can tell those from a link the post's own brackets formed once a
187    // tag beside them turned literal. The mark is stripped again before it
188    // returns.
189    let mark = link_mark.map(String::from).unwrap_or_default();
190    for (pattern, replacement) in [
191        (r"(?is)\[url=([^\]]+)\](.*?)\[/url\]", "[$2]($1)"),
192        (r"(?is)\[url\](.*?)\[/url\]", "<$1>"),
193        (r"(?is)\[email\](.*?)\[/email\]", "<mailto:$1>"),
194        (r"(?is)\[img(?:=[^\]]*)?\](.*?)\[/img\]", "![]($1)"),
195    ] {
196        text = replace(text, pattern, &format!("{mark}{replacement}"));
197    }
198    text = convert_marks(&text, link_mark);
199    let styling = re(r"(?is)\[(size|color|font)=([^\]]*)\](.*?)\[/(?:size|color|font)\]");
200    text = styling
201        .replace_all(&text, |caps: &Captures<'_>| {
202            styling_drops.push((caps[1].to_ascii_lowercase(), caps[2].to_string()));
203            caps[3].to_string()
204        })
205        .into_owned();
206    for (pattern, replacement) in [
207        (
208            r"(?is)\[(?:center|left|right)\](.*?)\[/(?:center|left|right)\]",
209            "$1",
210        ),
211        (r"(?is)\[(?:c|icode)\](.*?)\[/(?:c|icode)\]", "`$1`"),
212        (
213            r"(?i)\[youtube\]([a-z0-9_-]+)\[/youtube\]",
214            "![YouTube Video](https://www.youtube.com/watch?v=$1)",
215        ),
216    ] {
217        text = replace(text, pattern, replacement);
218    }
219    // Forced brace form: a sup/sub is often intraword (E=mc[sup]2[/sup]). An
220    // empty one has no spelling and goes (ruling markup-carve/carve-rs#1719).
221    for (pattern, open, close) in [
222        (r"(?is)\[sup\](.*?)\[/sup\]", "{^", "^}"),
223        (r"(?is)\[sub\](.*?)\[/sub\]", "{,", ",}"),
224    ] {
225        text = re(pattern)
226            .replace_all(&text, |caps: &Captures<'_>| {
227                if caps[1].is_empty() {
228                    String::new()
229                } else {
230                    format!("{open}{}{close}", &caps[1])
231                }
232            })
233            .into_owned();
234    }
235    text
236}
237
238/// One piece of a formatting-tag tree: text, or a tag by its arena index.
239enum MarkItem {
240    Text(String),
241    Mark(usize),
242}
243
244struct ParsedMark {
245    kind: u8,
246    open: String,
247    children: Vec<MarkItem>,
248    unclosed: bool,
249}
250
251fn mark_delim(kind: u8) -> char {
252    match kind {
253        b'b' => '*',
254        b'i' => '/',
255        b'u' => '_',
256        _ => '~',
257    }
258}
259
260/// The formatting tags as a tree, kept in an arena so neither building nor
261/// dropping it recurses on the author's nesting depth. A close tag that does
262/// not match the innermost open one stays literal text, and so does an open
263/// tag never closed (marked here, unwound by `flatten_same_kind`).
264fn parse_marks(text: &str) -> (Vec<ParsedMark>, Vec<MarkItem>) {
265    let mut arena: Vec<ParsedMark> = Vec::new();
266    let mut root: Vec<MarkItem> = Vec::new();
267    let mut stack: Vec<usize> = Vec::new();
268    let mut from = 0;
269    for caps in re(r"(?i)\[(/?)(b|i|u|s)\]").captures_iter(text) {
270        let whole = caps.get(0).expect("group 0 always matches");
271        let kind = caps[2].to_ascii_lowercase().as_bytes()[0];
272        let lead = MarkItem::Text(text[from..whole.start()].to_string());
273        match stack.last() {
274            Some(&top) => arena[top].children.push(lead),
275            None => root.push(lead),
276        }
277        from = whole.end();
278        if caps[1].is_empty() {
279            arena.push(ParsedMark {
280                kind,
281                open: whole.as_str().to_string(),
282                children: Vec::new(),
283                unclosed: false,
284            });
285            let id = arena.len() - 1;
286            match stack.last() {
287                Some(&top) => arena[top].children.push(MarkItem::Mark(id)),
288                None => root.push(MarkItem::Mark(id)),
289            }
290            stack.push(id);
291        } else if stack.last().is_some_and(|&top| arena[top].kind == kind) {
292            stack.pop();
293        }
294        // A close tag that matches no open one is dropped here rather than by
295        // cleanup(): left in, it would be the content of the tag around it, and
296        // that tag written as a pair around nothing once cleanup() took it.
297    }
298    let tail = MarkItem::Text(text[from..].to_string());
299    match stack.last() {
300        Some(&top) => arena[top].children.push(tail),
301        None => root.push(tail),
302    }
303    for id in stack {
304        arena[id].unclosed = true;
305    }
306    (arena, root)
307}
308
309/// A tag after flattening: its kind and the arena slot holding its children.
310struct FlatMark {
311    kind: u8,
312    slot: usize,
313}
314
315enum FlatItem {
316    Text(String),
317    Mark(FlatMark),
318}
319
320/// An unclosed tag becomes its literal text followed by its content. Carve has
321/// no second level of one kind (E3), so a tag inside an open tag of its own
322/// kind adds nothing and is replaced by its content. Iterative, and a
323/// transparent tag writes straight into its parent's slot, so this is linear;
324/// with four kinds the tree left behind is at most four deep, which keeps
325/// `write_marks` recursion bounded.
326fn flatten_same_kind(mut arena: Vec<ParsedMark>, root: Vec<MarkItem>) -> Vec<Vec<FlatItem>> {
327    let mut slots: Vec<Vec<FlatItem>> = vec![Vec::new()];
328    // (items still to place, slot they go to, kinds open around them)
329    let mut frames: Vec<(std::vec::IntoIter<MarkItem>, usize, [bool; 4])> =
330        vec![(root.into_iter(), 0, [false; 4])];
331    let kind_index = |kind: u8| match kind {
332        b'b' => 0,
333        b'i' => 1,
334        b'u' => 2,
335        _ => 3,
336    };
337    while let Some((items, slot, open)) = frames.last_mut() {
338        let Some(item) = items.next() else {
339            frames.pop();
340            continue;
341        };
342        let (slot, open) = (*slot, *open);
343        match item {
344            MarkItem::Text(text) => slots[slot].push(FlatItem::Text(text)),
345            MarkItem::Mark(id) => {
346                let children = std::mem::take(&mut arena[id].children);
347                let kind = arena[id].kind;
348                if arena[id].unclosed {
349                    slots[slot].push(FlatItem::Text(std::mem::take(&mut arena[id].open)));
350                    frames.push((children.into_iter(), slot, open));
351                } else if open[kind_index(kind)] {
352                    frames.push((children.into_iter(), slot, open));
353                } else {
354                    slots.push(Vec::new());
355                    let child_slot = slots.len() - 1;
356                    slots[slot].push(FlatItem::Mark(FlatMark {
357                        kind,
358                        slot: child_slot,
359                    }));
360                    let mut inner = open;
361                    inner[kind_index(kind)] = true;
362                    frames.push((children.into_iter(), child_slot, inner));
363                }
364            }
365        }
366    }
367    slots
368}
369
370// The tree is at most four deep, so this recursion is bounded.
371fn slot_has_content(slots: &[Vec<FlatItem>], slot: usize) -> bool {
372    slots[slot].iter().any(|item| match item {
373        FlatItem::Text(text) => !text.is_empty(),
374        FlatItem::Mark(mark) => slot_has_content(slots, mark.slot),
375    })
376}
377
378/// For each item, the first character written after it, or `None` when nothing
379/// is. One pass from the right, so a run of empty tags costs nothing per tag.
380fn next_chars(slots: &[Vec<FlatItem>], slot: usize) -> Vec<Option<char>> {
381    let items = &slots[slot];
382    let mut next = vec![None; items.len()];
383    let mut after = None;
384    for (k, item) in items.iter().enumerate().rev() {
385        next[k] = after;
386        match item {
387            FlatItem::Text(text) => {
388                if let Some(first) = text.chars().next() {
389                    after = Some(first);
390                }
391            }
392            FlatItem::Mark(mark) => {
393                if slot_has_content(slots, mark.slot) {
394                    after = Some('{');
395                }
396            }
397        }
398    }
399    next
400}
401
402fn is_word(c: Option<char>) -> bool {
403    c.is_some_and(|c| c.is_ascii_alphanumeric() || c == '_')
404}
405
406/// Write the tree the way the Carve writer would. An empty tag holds nothing a
407/// reader sees and has no spelling, so it goes (ruling markup-carve/carve-rs#1719).
408/// A bare pair the CARVE-P3-013 guards would not read back, or that the writer
409/// would brace, takes the braced form, as does a `/` around content that would
410/// read back as bold-italic.
411/// Written text, with the byte span `[start, end)` of every formatting pair the
412/// writer put there itself.
413struct Written {
414    text: String,
415    marks: Vec<(usize, usize)>,
416}
417
418fn write_marks(
419    slots: &[Vec<FlatItem>],
420    slot: usize,
421    outer_next: Option<char>,
422    prev: Option<char>,
423) -> Written {
424    // Chunks rather than one growing string, so escaping the character before
425    // an opener rewrites only the text chunk holding it.
426    let mut parts: Vec<(String, Vec<(usize, usize)>)> = Vec::new();
427    let mut last = prev;
428    let mut text_part: Option<usize> = None;
429    let mut escape_brace = false;
430    let nexts = next_chars(slots, slot);
431    for (index, item) in slots[slot].iter().enumerate() {
432        match item {
433            FlatItem::Text(text) => {
434                if text.is_empty() {
435                    continue;
436                }
437                // A `{` before a bare opener and a `}` after its closer would read
438                // as the braced form, eating both braces; the writer escapes the `}`.
439                let chunk = if escape_brace && text.starts_with('}') {
440                    format!("\\{text}")
441                } else {
442                    text.clone()
443                };
444                last = chunk.chars().next_back();
445                parts.push((chunk, Vec::new()));
446                escape_brace = false;
447                text_part = Some(parts.len() - 1);
448            }
449            FlatItem::Mark(mark) => {
450                let delim = mark_delim(mark.kind);
451                // The parent's own delimiters are not text: a child at its edge
452                // has no neighbor there, which is how the writer spells `/_x_/`.
453                let inner = write_marks(slots, mark.slot, None, None);
454                let mut body = inner.text;
455                if body.is_empty() {
456                    continue;
457                }
458                // The post's own text was escaped while the tags were still tags,
459                // so a literal delimiter now touching an opener was never seen.
460                // Already escaped only behind an ODD run of backslashes.
461                if text_part.is_some()
462                    && text_part == parts.len().checked_sub(1)
463                    && last.is_some_and(|c| "*/_~=".contains(c))
464                {
465                    let chunk = &parts[parts.len() - 1].0;
466                    let run = chunk[..chunk.len() - 1]
467                        .bytes()
468                        .rev()
469                        .take_while(|b| *b == b'\\')
470                        .count();
471                    if run % 2 == 0 {
472                        let chunk = &mut parts.last_mut().expect("text_part is the last chunk").0;
473                        chunk.insert(chunk.len() - 1, '\\');
474                    }
475                }
476                let before = last;
477                let next = nexts[index].or(outer_next);
478                let ws = |c: char| matches!(c, ' ' | '\t' | '\r' | '\n');
479                let braced = body.starts_with(ws)
480                    || body.ends_with(ws)
481                    || is_word(before)
482                    || before == Some(delim)
483                    || (before == Some('/') && (delim == '/' || delim == '_'))
484                    // `#_x_` is a hashtag, `@_x_` a mention, `:_x_:` a symbol.
485                    || (delim == '_' && matches!(before, Some('#' | '@' | ':')))
486                    || is_word(next)
487                    || body.starts_with(delim)
488                    || body.ends_with(delim)
489                    || (delim == '/' && body.starts_with('*') && body.ends_with('*'));
490                if body.ends_with('{') {
491                    let backslashes = body.as_bytes()[..body.len() - 1]
492                        .iter()
493                        .rev()
494                        .take_while(|&&byte| byte == b'\\')
495                        .count();
496                    if backslashes % 2 == 0 {
497                        body.insert(body.len() - 1, '\\');
498                    }
499                }
500                let (open, close) = if braced {
501                    (format!("{{{delim}"), format!("{delim}}}"))
502                } else {
503                    (delim.to_string(), delim.to_string())
504                };
505                let chunk = format!("{open}{body}{close}");
506                let mut marks = vec![(0, chunk.len())];
507                marks.extend(
508                    inner
509                        .marks
510                        .iter()
511                        .map(|&(from, to)| (open.len() + from, open.len() + to)),
512                );
513                last = chunk.chars().next_back();
514                parts.push((chunk, marks));
515                escape_brace = !braced && before == Some('{');
516            }
517        }
518    }
519    let mut text = String::new();
520    let mut marks = Vec::new();
521    for (chunk, chunk_marks) in parts {
522        marks.extend(
523            chunk_marks
524                .iter()
525                .map(|&(from, to)| (text.len() + from, text.len() + to)),
526        );
527        text.push_str(&chunk);
528    }
529    Written { text, marks }
530}
531
532const REPAIR_ROUNDS: usize = 16;
533
534/// Inline constructs bbcode has no way to ask for at this stage, by AST-JSON
535/// type. Links, images and autolinks are judged by the written-link mark
536/// instead, and the rest is prose.
537fn is_unwritten(kind: &str) -> bool {
538    !matches!(
539        kind,
540        "text"
541            | "soft_break"
542            | "hard_break"
543            | "escaped_text"
544            | "smart_punctuation"
545            | "link"
546            | "image"
547            | "autolink"
548            | "strong"
549            | "emphasis"
550            | "underline"
551            | "strike"
552            | "document"
553            | "paragraph"
554    )
555}
556
557/// Block tags a later pass turns into structure; `sup` and `sub` become a
558/// braced span, which is just as closed to its neighbors.
559fn is_later_block_tag(name: &str) -> bool {
560    matches!(
561        name.to_ascii_lowercase().as_str(),
562        "*" | "quote"
563            | "list"
564            | "code"
565            | "c"
566            | "icode"
567            | "sup"
568            | "sub"
569            | "center"
570            | "left"
571            | "right"
572            | "youtube"
573            | "table"
574            | "tr"
575            | "td"
576            | "th"
577            | "noparse"
578            | "hr"
579    )
580}
581
582/// The text as the later passes will leave it around inline content, so the
583/// repair parse sees the neighbors the reader will: a close tag or a valued
584/// open tag is deleted by cleanup(), which joins the text either side of it,
585/// and a block tag becomes structure, which separates it. A formatting tag left
586/// unclosed, or one this pass does not know, stays the literal text it is.
587/// `origin` maps each byte of the copy, and one past its end, to `text`.
588fn as_later_passes_leave_it(text: &str) -> (String, Vec<usize>) {
589    let mut copy = String::with_capacity(text.len());
590    let mut origin = Vec::with_capacity(text.len() + 1);
591    let mut from = 0;
592    // The second branch is a quote opener with its fields after a space (`[quote name="bob"]`).
593    for caps in re(r"(?i)\[(/?)(\*|[a-z][a-z0-9]*)(=[^\]\n]*)?\]|\[(quote)( [^\]\n]*)\]")
594        .captures_iter(text)
595    {
596        let whole = caps.get(0).expect("group 0 always matches");
597        let at = whole.start();
598        if at > 0 && text.as_bytes()[at - 1] == b'\\' {
599            continue;
600        }
601        let removed = caps.get(4).is_some() || !caps[1].is_empty() || caps.get(3).is_some();
602        if !removed && !is_later_block_tag(&caps[2]) {
603            continue;
604        }
605        origin.extend(from..at);
606        copy.push_str(&text[from..at]);
607        if !removed {
608            origin.extend(at..whole.end());
609            copy.push_str(&"\u{1}".repeat(whole.len()));
610        }
611        from = whole.end();
612    }
613    origin.extend(from..text.len());
614    copy.push_str(&text[from..]);
615    origin.push(text.len());
616    (copy, origin)
617}
618
619/// The post's text was escaped while the tags were still tags, so once they are
620/// delimiters a literal character can combine with them, or with text a dropped
621/// tag brought together, into a construct nobody wrote: `#[/i]x` reads as the
622/// hashtag `#x`, and `~}[s]x[/s]` as a strikethrough of `}`. Rather than predict
623/// every such construct, parse the result and escape the first character of
624/// each one that is not a written pair, until none is left. A written pair that
625/// closes early was closed by a literal delimiter inside it, and that delimiter
626/// is the one escaped. A link without the written-link mark came from the
627/// post's own brackets.
628fn repair_unwritten_constructs(written: Written, link_mark: Option<char>) -> String {
629    let Written {
630        mut text,
631        mut marks,
632    } = written;
633    for _ in 0..REPAIR_ROUNDS {
634        let (copy, origin) = as_later_passes_leave_it(&text);
635        // Codepoint offset in the copy -> byte in `text`.
636        let mut bytes: Vec<usize> = copy.char_indices().map(|(at, _)| at).collect();
637        bytes.push(copy.len());
638        let index = |offset: u64| origin[bytes[(offset as usize).min(bytes.len() - 1)]];
639        let pairs: std::collections::HashMap<usize, usize> = marks.iter().copied().collect();
640        // A written pair's own delimiters are never escaped: when a literal
641        // delimiter closes a pair early, the parse finds a second span starting
642        // at the pair's real closer, and escaping that would hand the following
643        // text to the pair. The early closer is escaped in this same round, so
644        // the next one reads the pair whole.
645        let mut written = std::collections::HashSet::new();
646        for &(from, to) in &marks {
647            let width = if text[from..].starts_with('{') { 2 } else { 1 };
648            for k in 0..width {
649                written.insert(from + k);
650                written.insert(to - 1 - k);
651            }
652        }
653        let mut escape_at = std::collections::BTreeSet::new();
654        // Positions are off by default, and the repair is all offsets.
655        let parsed =
656            crate::parse_with_options(&copy, &crate::Options::default().with_positions(true));
657        let tree: serde_json::Value =
658            serde_json::from_str(&crate::to_json(&parsed)).expect("the AST encodes");
659        let mut stack = vec![&tree];
660        while let Some(node) = stack.pop() {
661            if let Some(children) = node.get("children").and_then(|c| c.as_array()) {
662                stack.extend(children.iter());
663            }
664            let Some(kind) = node.get("type").and_then(|t| t.as_str()) else {
665                continue;
666            };
667            let Some(pos) = node.get("pos") else {
668                continue;
669            };
670            let (Some(start), Some(end)) = (
671                pos.get("startOffset").and_then(|v| v.as_u64()),
672                pos.get("endOffset").and_then(|v| v.as_u64()),
673            ) else {
674                continue;
675            };
676            let at = index(start);
677            match kind {
678                "strong" | "emphasis" | "underline" | "strike" => match pairs.get(&at) {
679                    None => {
680                        escape_at.insert(at);
681                    }
682                    Some(&written_end) => {
683                        let end = index(end.saturating_sub(1)) + 1;
684                        if end < written_end {
685                            escape_at.insert(end - 1);
686                        }
687                    }
688                },
689                "link" | "image" | "autolink" => {
690                    let marked = link_mark.is_some_and(|mark| text[..at].ends_with(mark));
691                    if !marked {
692                        escape_at.insert(if kind == "image" { at + 1 } else { at });
693                    }
694                }
695                _ if is_unwritten(kind) => {
696                    escape_at.insert(at);
697                }
698                _ => {}
699            }
700        }
701        escape_at.retain(|at| !written.contains(at));
702        if escape_at.is_empty() {
703            break;
704        }
705        let cuts: Vec<usize> = escape_at.into_iter().collect();
706        let mut out = String::with_capacity(text.len() + cuts.len());
707        let mut copied = 0;
708        for &at in &cuts {
709            out.push_str(&text[copied..at]);
710            out.push('\\');
711            copied = at;
712        }
713        out.push_str(&text[copied..]);
714        text = out;
715        // Each offset moves by the number of escapes inserted before it.
716        let shift = |k: usize| k + cuts.partition_point(|&cut| cut < k);
717        marks = marks
718            .iter()
719            .map(|&(from, to)| (shift(from), shift(to)))
720            .collect();
721    }
722    text
723}
724
725fn convert_marks(text: &str, link_mark: Option<char>) -> String {
726    let (arena, root) = parse_marks(text);
727    let slots = flatten_same_kind(arena, root);
728    let repaired = repair_unwritten_constructs(write_marks(&slots, 0, None, None), link_mark);
729    match link_mark {
730        Some(mark) => repaired.replace(mark, ""),
731        None => repaired,
732    }
733}
734
735fn convert_code(mut text: String) -> String {
736    text = re(r"(?is)\[code=([^\]]+)\](.*?)\[/code\]")
737        .replace_all(&text, |caps: &Captures<'_>| {
738            let language = caps[1].trim().to_ascii_lowercase();
739            let language = language.trim_start_matches('=').trim_start();
740            format!("\n\n```{language}\n{}\n```\n\n", caps[2].trim())
741        })
742        .into_owned();
743    text = re(r"(?is)\[code\](.*?)\[/code\]")
744        .replace_all(&text, |caps: &Captures<'_>| {
745            format!("\n\n```\n{}\n```\n\n", caps[1].trim())
746        })
747        .into_owned();
748    text
749}
750
751fn convert_other(mut text: String) -> String {
752    text = re(r"(?is)\[spoiler(?:=([^\]]+))?\](.*?)\[/spoiler\]")
753        .replace_all(&text, |caps: &Captures<'_>| {
754            let title = caps.get(1).map(|m| m.as_str().trim()).unwrap_or("");
755            let title = title.replace('\\', "\\\\").replace('"', "\\\"");
756            let attr = if title.is_empty() {
757                String::new()
758            } else {
759                format!("{{title=\"{title}\"}}\n")
760            };
761            format!("{attr}::: spoiler\n{}\n:::\n", caps[2].trim())
762        })
763        .into_owned();
764
765    let table = re(r"(?is)\[table\](.*?)\[/table\]");
766    text = table
767        .replace_all(&text, |caps: &Captures<'_>| {
768            let rows = re(r"(?is)\[tr\](.*?)\[/tr\]");
769            let mut output = Vec::new();
770            for row in rows.captures_iter(&caps[1]) {
771                let body = &row[1];
772                let headers = re(r"(?is)\[th\](.*?)\[/th\]")
773                    .captures_iter(body)
774                    .map(|cell| format!("|= {}", cell[1].trim()))
775                    .collect::<Vec<_>>();
776                if !headers.is_empty() {
777                    output.push(format!("{} |", headers.join(" ")));
778                    continue;
779                }
780                let cells = re(r"(?is)\[td\](.*?)\[/td\]")
781                    .captures_iter(body)
782                    .map(|cell| cell[1].trim().to_string())
783                    .collect::<Vec<_>>();
784                if !cells.is_empty() {
785                    output.push(format!("| {} |", cells.join(" | ")));
786                }
787            }
788            format!("\n\n{}\n\n", output.join("\n"))
789        })
790        .into_owned();
791    text
792}
793
794fn quote_block(content: &str, author: Option<&str>) -> String {
795    let body = content.trim();
796    let quoted = if body.is_empty() {
797        ">".to_string()
798    } else {
799        body.lines()
800            .map(|line| {
801                if line.trim().is_empty() {
802                    ">".to_string()
803                } else {
804                    format!("> {line}")
805                }
806            })
807            .collect::<Vec<_>>()
808            .join("\n")
809    };
810    let attribution = format_attribution(author.unwrap_or(""));
811    if attribution.is_empty() {
812        format!("\n\n{quoted}\n\n")
813    } else {
814        format!("\n\n{quoted}\n^ {attribution}\n\n")
815    }
816}
817
818/// phpBB stores `time` as Unix seconds; 9 to 11 digits is that, a shorter number a clock value.
819fn unix_seconds_date(value: &str) -> Option<String> {
820    if !(9..=11).contains(&value.len()) || !value.bytes().all(|byte| byte.is_ascii_digit()) {
821        return None;
822    }
823    let days = value.parse::<i64>().ok()? / 86_400 + 719_468;
824    let era = days.div_euclid(146_097);
825    let day_of_era = days.rem_euclid(146_097);
826    let year_of_era =
827        (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
828    let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
829    let month_parameter = (5 * day_of_year + 2) / 153;
830    let day = day_of_year - (153 * month_parameter + 2) / 5 + 1;
831    let month = month_parameter + if month_parameter < 10 { 3 } else { -9 };
832    let year = year_of_era + era * 400 + i64::from(month <= 2);
833    Some(format!("{year:04}-{month:02}-{day:02}"))
834}
835
836fn format_attribution(source: &str) -> String {
837    format_attribution_with_losses(source).0
838}
839
840#[derive(Debug, Clone, PartialEq, Eq)]
841pub(crate) struct DroppedQuoteField {
842    pub field: String,
843    pub unnamed: bool,
844    pub line: usize,
845}
846
847enum AttributionPart {
848    Field { key: String, value: String },
849    Text { text: String, quoted: bool },
850}
851
852/// ASCII whitespace only, so every engine splits an attribution at the same places.
853fn is_attribution_space(byte: u8) -> bool {
854    matches!(byte, b' ' | b'\t' | b'\n' | b'\r' | b'\x0c')
855}
856
857/// Split an attribution into `key=value` fields, quoted texts and bare words.
858fn attribution_parts(attribution: &str) -> Vec<AttributionPart> {
859    let bytes = attribution.as_bytes();
860    let run_end = |from: usize| {
861        let mut at = from;
862        while at < bytes.len() && !is_attribution_space(bytes[at]) {
863            at += 1;
864        }
865        at
866    };
867    let quoted_end = |from: usize| -> Option<usize> {
868        let quote = *bytes.get(from)?;
869        if quote != b'"' && quote != b'\'' {
870            return None;
871        }
872        bytes[from + 1..]
873            .iter()
874            .position(|&byte| byte == quote)
875            .map(|offset| from + 1 + offset)
876    };
877    let key = re(r"^[A-Za-z_][A-Za-z0-9_-]*=");
878    let mut parts = Vec::new();
879    let mut i = 0;
880    while i < bytes.len() {
881        if is_attribution_space(bytes[i]) {
882            i += 1;
883            continue;
884        }
885        if let Some(found) = key.find(&attribution[i..]) {
886            let from = i + found.end();
887            let name = attribution[i..from - 1].to_string();
888            if let Some(close) = quoted_end(from) {
889                parts.push(AttributionPart::Field {
890                    key: name,
891                    value: attribution[from + 1..close].to_string(),
892                });
893                i = close + 1;
894            } else {
895                let to = run_end(from);
896                parts.push(AttributionPart::Field {
897                    key: name,
898                    value: attribution[from..to].to_string(),
899                });
900                i = to;
901            }
902            continue;
903        }
904        if let Some(close) = quoted_end(i) {
905            parts.push(AttributionPart::Text {
906                text: attribution[i + 1..close].to_string(),
907                quoted: true,
908            });
909            i = close + 1;
910            continue;
911        }
912        let to = run_end(i);
913        parts.push(AttributionPart::Text {
914            text: attribution[i..to].to_string(),
915            quoted: false,
916        });
917        i = to;
918    }
919    parts
920}
921
922fn all_ascii_digits(value: &str) -> bool {
923    !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit())
924}
925
926/// Read a quote attribution into its `name (date) #id` caption and the parts it
927/// drops, in source order. docs/format-bridges.md, "BBCode quote attribution
928/// reading", is the contract.
929fn format_attribution_with_losses(source: &str) -> (String, Vec<(String, bool)>) {
930    let parts = attribution_parts(source);
931    // (source part index, field or text, unnamed)
932    let mut drops: Vec<(usize, String, bool)> = Vec::new();
933    let mut first: std::collections::HashMap<&'static str, (usize, String, String)> =
934        std::collections::HashMap::new();
935    let mut texts: Vec<(usize, String, bool)> = Vec::new();
936    for (index, part) in parts.into_iter().enumerate() {
937        match part {
938            AttributionPart::Text { text, quoted } => {
939                if !text.is_empty() {
940                    texts.push((index, text, quoted));
941                }
942            }
943            AttributionPart::Field { key, value } => {
944                let kind = match key.to_ascii_lowercase().as_str() {
945                    "name" | "author" => Some("name"),
946                    "post_id" => Some("post_id"),
947                    "id" => Some("id"),
948                    "date" => Some("date"),
949                    "time" => Some("time"),
950                    _ => None,
951                };
952                let usable = !value.is_empty()
953                    && (!matches!(kind, Some("post_id" | "id")) || all_ascii_digits(&value));
954                match kind {
955                    Some(kind) if usable && !first.contains_key(kind) => {
956                        first.insert(kind, (index, key, value));
957                    }
958                    _ => drops.push((index, key, false)),
959                }
960            }
961        }
962    }
963
964    let leading_id = match texts.first() {
965        Some((0, text, true)) if all_ascii_digits(text) => Some(texts.remove(0)),
966        _ => None,
967    };
968    let mut id = None;
969    if let Some((_, _, value)) = first.get("post_id") {
970        id = Some(value.clone());
971        if let Some((index, key, _)) = first.get("id") {
972            drops.push((*index, key.clone(), false));
973        }
974    } else if let Some((_, _, value)) = first.get("id") {
975        id = Some(value.clone());
976    } else if let Some((_, text, _)) = &leading_id {
977        id = Some(text.clone());
978    }
979    // A leading `"9"` that a field outranks is reported under the field it stands for.
980    if let Some((index, _, _)) = &leading_id {
981        if first.contains_key("post_id") || first.contains_key("id") {
982            drops.push((*index, "id".to_string(), false));
983        }
984    }
985
986    let mut text = texts
987        .iter()
988        .map(|(_, text, _)| text.as_str())
989        .collect::<Vec<_>>()
990        .join(" ");
991    let mut suffix = None;
992    if id.is_none() {
993        if let Some(caps) = re(r"^(?s)(.+);([0-9]+)$").captures(&text) {
994            id = Some(caps[2].to_string());
995            suffix = Some(format!(";{}", &caps[2]));
996            text = caps[1].to_string();
997        }
998    }
999
1000    let name = match first.get("name") {
1001        Some((_, _, value)) => {
1002            let mut run: Option<(usize, String, usize)> = None;
1003            for (index, value, quoted) in &texts {
1004                if *quoted {
1005                    if let Some((start, words, _)) = run.take() {
1006                        drops.push((start, words, true));
1007                    }
1008                    drops.push((*index, value.clone(), true));
1009                } else if let Some((_, words, last)) = &mut run {
1010                    if *index == *last + 1 {
1011                        words.push(' ');
1012                        words.push_str(value);
1013                        *last = *index;
1014                    } else {
1015                        let (start, words, _) =
1016                            run.replace((*index, value.clone(), *index)).unwrap();
1017                        drops.push((start, words, true));
1018                    }
1019                } else {
1020                    run = Some((*index, value.clone(), *index));
1021                }
1022            }
1023            if let Some((start, words, _)) = run {
1024                drops.push((start, words, true));
1025            }
1026            if let Some(suffix) = suffix {
1027                if let Some(last_index) = texts.last().map(|part| part.0) {
1028                    if let Some(position) = drops
1029                        .iter()
1030                        .rposition(|(index, _, unnamed)| *unnamed && *index <= last_index)
1031                    {
1032                        if drops[position].1.ends_with(&suffix) {
1033                            let length = drops[position].1.len() - suffix.len();
1034                            drops[position].1.truncate(length);
1035                            if drops[position].1.is_empty() {
1036                                drops.remove(position);
1037                            }
1038                        }
1039                    }
1040                }
1041            }
1042            value.clone()
1043        }
1044        None => text,
1045    };
1046
1047    let mut date = first.get("date").map(|(_, _, value)| value.clone());
1048    if let Some((_, _, time)) = first.get("time") {
1049        date = Some(match unix_seconds_date(time) {
1050            Some(day) => {
1051                if let Some((index, key, _)) = first.get("date") {
1052                    drops.push((*index, key.clone(), false));
1053                }
1054                day
1055            }
1056            None => match date {
1057                Some(date) => format!("{date} {time}"),
1058                None => time.clone(),
1059            },
1060        });
1061    }
1062
1063    let mut caption = Vec::new();
1064    if !name.is_empty() {
1065        caption.push(name);
1066    }
1067    if let Some(date) = date {
1068        caption.push(format!("({date})"));
1069    }
1070    if let Some(id) = id {
1071        caption.push(format!("#{id}"));
1072    }
1073    drops.sort_by_key(|(index, _, _)| *index);
1074    (
1075        caption.join(" "),
1076        drops
1077            .into_iter()
1078            .map(|(_, field, unnamed)| (field, unnamed))
1079            .collect(),
1080    )
1081}
1082
1083/// Each opening quote tag's attribution with the fields it dropped (field, unnamed).
1084type QuoteTagDrops = (String, Vec<(String, bool)>);
1085
1086fn convert_quotes(text: String) -> (String, Vec<QuoteTagDrops>) {
1087    let open = re(r"(?i)^\[quote(?:[= ]([^\]]*))?\]");
1088    let close = re(r"(?i)^\[/quote\]");
1089    let mut contents = vec![String::new()];
1090    let mut authors: Vec<Option<String>> = vec![None];
1091    let mut drops = Vec::new();
1092    let mut i = 0;
1093    while i < text.len() {
1094        let rest = &text[i..];
1095        if let Some(caps) = open.captures(rest) {
1096            let whole = caps.get(0).expect("whole quote opener");
1097            let attribution = caps.get(1).map_or("", |m| m.as_str());
1098            drops.push((
1099                attribution.to_string(),
1100                format_attribution_with_losses(attribution).1,
1101            ));
1102            contents.push(String::new());
1103            authors.push(caps.get(1).map(|m| m.as_str().to_string()));
1104            i += whole.end();
1105            continue;
1106        }
1107        if let Some(found) = close.find(rest) {
1108            i += found.end();
1109            if contents.len() > 1 {
1110                let content = contents.pop().expect("nested quote content");
1111                let author = authors.pop().expect("nested quote author");
1112                let block = quote_block(&content, author.as_deref());
1113                contents
1114                    .last_mut()
1115                    .expect("root quote buffer")
1116                    .push_str(&block);
1117            }
1118            continue;
1119        }
1120        let ch = rest.chars().next().expect("non-empty remainder");
1121        contents.last_mut().expect("quote buffer").push(ch);
1122        i += ch.len_utf8();
1123    }
1124    while contents.len() > 1 {
1125        let content = contents.pop().expect("nested quote content");
1126        let author = authors.pop().expect("nested quote author");
1127        let block = quote_block(&content, author.as_deref());
1128        contents
1129            .last_mut()
1130            .expect("root quote buffer")
1131            .push_str(&block);
1132    }
1133    (contents.pop().expect("root quote buffer"), drops)
1134}
1135
1136#[derive(Debug)]
1137struct ListFrame {
1138    ordered: bool,
1139    marker: char,
1140    delimiter: char,
1141    lead: String,
1142    items: Vec<String>,
1143    current: Option<String>,
1144    sibling_index: usize,
1145}
1146
1147fn trim_item(body: &str) -> String {
1148    body.lines()
1149        .map(|line| line.trim_end())
1150        .collect::<Vec<_>>()
1151        .join("\n")
1152        .trim()
1153        .to_string()
1154}
1155
1156fn render_list(frame: &ListFrame) -> String {
1157    frame
1158        .items
1159        .iter()
1160        .enumerate()
1161        .map(|(index, item)| {
1162            let marker = if frame.ordered {
1163                format!("{}{} ", index + 1, frame.delimiter)
1164            } else {
1165                format!("{} ", frame.marker)
1166            };
1167            let indent = " ".repeat(marker.len());
1168            trim_item(item)
1169                .lines()
1170                .enumerate()
1171                .map(|(line_index, line)| {
1172                    if line_index == 0 {
1173                        format!("{marker}{line}")
1174                    } else if line.is_empty() {
1175                        String::new()
1176                    } else {
1177                        format!("{indent}{line}")
1178                    }
1179                })
1180                .collect::<Vec<_>>()
1181                .join("\n")
1182        })
1183        .collect::<Vec<_>>()
1184        .join("\n")
1185}
1186
1187fn convert_lists(text: String) -> String {
1188    let open = re(r"(?i)^\[list(?:=([^\]]*))?\]");
1189    let item = re(r"(?i)^\[\*\]");
1190    let close = re(r"(?i)^\[/list\]");
1191    let mut frames: Vec<ListFrame> = Vec::new();
1192    let mut output = String::new();
1193    let mut root_sibling = 0usize;
1194    let mut i = 0usize;
1195
1196    let append = |frames: &mut Vec<ListFrame>, output: &mut String, value: &str| {
1197        if let Some(frame) = frames.last_mut() {
1198            if let Some(current) = frame.current.as_mut() {
1199                current.push_str(value);
1200            } else {
1201                frame.lead.push_str(value);
1202            }
1203        } else {
1204            output.push_str(value);
1205        }
1206    };
1207
1208    while i < text.len() {
1209        let rest = &text[i..];
1210        if let Some(caps) = open.captures(rest) {
1211            let value = caps.get(1).map(|m| m.as_str().trim());
1212            if value.is_none() || value == Some("1") {
1213                let axis = if let Some(parent) = frames.last_mut() {
1214                    let axis = parent.sibling_index;
1215                    parent.sibling_index += 1;
1216                    axis
1217                } else {
1218                    let axis = root_sibling;
1219                    root_sibling += 1;
1220                    axis
1221                };
1222                frames.push(ListFrame {
1223                    ordered: value.is_some(),
1224                    marker: if axis % 2 == 0 { '-' } else { '*' },
1225                    delimiter: if axis % 2 == 0 { '.' } else { ')' },
1226                    lead: String::new(),
1227                    items: Vec::new(),
1228                    current: None,
1229                    sibling_index: 0,
1230                });
1231                i += caps.get(0).expect("list opener").end();
1232                continue;
1233            }
1234        }
1235        if !frames.is_empty() && item.is_match(rest) {
1236            let frame = frames.last_mut().expect("open list frame");
1237            if let Some(current) = frame.current.take() {
1238                frame.items.push(current);
1239            }
1240            frame.current = Some(String::new());
1241            frame.sibling_index = 0;
1242            i += 3;
1243            continue;
1244        }
1245        if !frames.is_empty() && close.is_match(rest) {
1246            i += "[/list]".len();
1247            close_list_frame(&mut frames, &mut output, &append);
1248            continue;
1249        }
1250        let ch = rest.chars().next().expect("non-empty remainder");
1251        let mut encoded = [0; 4];
1252        append(&mut frames, &mut output, ch.encode_utf8(&mut encoded));
1253        i += ch.len_utf8();
1254    }
1255    while !frames.is_empty() {
1256        close_list_frame(&mut frames, &mut output, &append);
1257    }
1258    output
1259}
1260
1261fn close_list_frame<F>(frames: &mut Vec<ListFrame>, output: &mut String, append: &F)
1262where
1263    F: Fn(&mut Vec<ListFrame>, &mut String, &str),
1264{
1265    let mut frame = frames.pop().expect("open list frame");
1266    if let Some(current) = frame.current.take() {
1267        frame.items.push(current);
1268    }
1269    let lead = trim_item(&frame.lead);
1270    let list = render_list(&frame);
1271    let block = if lead.is_empty() {
1272        list
1273    } else {
1274        format!("{lead}\n{list}")
1275    };
1276    if frames.is_empty() {
1277        output.push_str(&format!("\n\n{}\n\n", block.trim_end()));
1278    } else {
1279        let needs_newline = frames
1280            .last()
1281            .and_then(|parent| parent.current.as_ref())
1282            .is_some_and(|current| !current.is_empty() && !current.ends_with('\n'));
1283        if needs_newline {
1284            append(frames, output, "\n");
1285        }
1286        append(frames, output, block.trim_end());
1287        append(frames, output, "\n");
1288    }
1289}
1290
1291fn cleanup(mut text: String) -> String {
1292    text = replace(text, r"(?i)\[hr\]", "\n---\n");
1293    text = replace(text, r"(?i)\[/[a-z][a-z0-9]*\]", "");
1294    text = replace(text, r"(?i)\[[a-z][a-z0-9]*=[^\]]*\]", "");
1295    text = replace(text, r"\n{3,}", "\n\n");
1296    format!("{}\n", text.trim())
1297}
1298
1299/// Convert one bounded BBCode post to canonical Carve source.
1300pub fn bbcode_to_carve(source: &str) -> Result<String, BbcodeImportError> {
1301    bbcode_to_carve_with_drops(source).map(|(value, _)| value)
1302}
1303
1304pub(crate) fn bbcode_to_carve_with_drops(
1305    source: &str,
1306) -> Result<(String, Vec<DroppedQuoteField>), BbcodeImportError> {
1307    bbcode_to_carve_with_all_drops(source).map(|converted| (converted.value, converted.quote_drops))
1308}
1309
1310pub(crate) struct BbcodeConversion {
1311    pub value: String,
1312    pub quote_drops: Vec<DroppedQuoteField>,
1313    pub styling_drops: Vec<(String, String)>,
1314}
1315
1316pub(crate) fn bbcode_to_carve_with_all_drops(
1317    source: &str,
1318) -> Result<BbcodeConversion, BbcodeImportError> {
1319    if source.len() > BBCODE_MAX_INPUT_LENGTH {
1320        return Err(BbcodeImportError::InputTooLarge {
1321            bytes: source.len(),
1322            maximum: BBCODE_MAX_INPUT_LENGTH,
1323        });
1324    }
1325    let normalized = source
1326        .replace('\0', "\u{fffd}")
1327        .replace("\r\n", "\n")
1328        .replace('\r', "\n");
1329    let escaped = escape_text(&normalized)?;
1330    let (mut text, open, close, stash) = stash_literals(escaped)?;
1331    // Only a post with a link or image to convert needs the mark.
1332    let link_mark = if re(r"(?i)\[(?:url|email|img)\b").is_match(&text) {
1333        Some(pick_mark(&text, 0xe020)?)
1334    } else {
1335        None
1336    };
1337    let mut styling_drops = Vec::new();
1338    text = convert_pairs(text, link_mark, &mut styling_drops);
1339    text = convert_code(text);
1340    let (converted, quote_drops) = convert_quotes(text);
1341    text = converted;
1342    text = convert_lists(text);
1343    text = convert_other(text);
1344    text = cleanup(text);
1345    let value = restore(text, open, close, &stash);
1346    Ok(BbcodeConversion {
1347        value,
1348        quote_drops: locate_quote_drops(source, quote_drops),
1349        styling_drops,
1350    })
1351}
1352
1353fn locate_quote_drops(source: &str, quote_drops: Vec<QuoteTagDrops>) -> Vec<DroppedQuoteField> {
1354    let mut visible = source.replace("\r\n", "\n").replace('\r', "\n");
1355    for pattern in [
1356        r"(?is)\[code(?:=[^\]]*)?\].*?\[/code\]",
1357        r"(?is)\[(?:c|icode)\].*?\[/(?:c|icode)\]",
1358        r"(?is)\[noparse\].*?\[/noparse\]",
1359    ] {
1360        let pattern = re(pattern);
1361        visible = pattern
1362            .replace_all(&visible, |caps: &Captures<'_>| {
1363                caps[0]
1364                    .chars()
1365                    .map(|ch| if ch == '\n' { '\n' } else { ' ' })
1366                    .collect::<String>()
1367            })
1368            .into_owned();
1369    }
1370    let quote = re(r"(?i)\[quote(?:[= ]([^\]]*))?\]");
1371    let mut tags = Vec::new();
1372    let mut line = 1;
1373    let mut scanned = 0;
1374    for caps in quote.captures_iter(&visible) {
1375        let matched = caps.get(0).expect("quote match");
1376        line += visible[scanned..matched.start()]
1377            .bytes()
1378            .filter(|byte| *byte == b'\n')
1379            .count();
1380        scanned = matched.start();
1381        let attribution = caps.get(1).map_or("", |m| m.as_str()).replace('\\', "");
1382        tags.push((line, attribution));
1383    }
1384    let mut next = 0;
1385    let mut result = Vec::new();
1386    for (attribution, dropped) in quote_drops {
1387        let wanted = attribution.replace('\\', "");
1388        let found = (next..tags.len()).find(|&at| tags[at].1 == wanted);
1389        let at = found.unwrap_or(next.min(tags.len().saturating_sub(1)));
1390        if let Some(found) = found {
1391            next = found + 1;
1392        }
1393        for (field, unnamed) in dropped {
1394            result.push(DroppedQuoteField {
1395                field,
1396                unnamed,
1397                line: tags.get(at).map_or(1, |tag| tag.0),
1398            });
1399        }
1400    }
1401    result
1402}
1403
1404#[cfg(test)]
1405mod tests {
1406    use super::*;
1407
1408    #[test]
1409    fn quote_field_detector_matches_attribution() {
1410        assert_eq!(
1411            format_attribution_with_losses("\"alice\" post_id=12 user_id=3").1,
1412            vec![("user_id".into(), false)]
1413        );
1414        assert_eq!(
1415            bbcode_to_carve_with_drops("[quote=alice user_id=3]x[/quote]")
1416                .unwrap()
1417                .1[0]
1418                .line,
1419            1
1420        );
1421    }
1422
1423    #[test]
1424    fn common_tags_convert() {
1425        assert_eq!(bbcode_to_carve("[b]b[/b] [i]i[/i]").unwrap(), "*b* /i/\n");
1426        assert_eq!(
1427            bbcode_to_carve("[url=https://e.test]x[/url]").unwrap(),
1428            "[x](https://e.test)\n"
1429        );
1430        assert_eq!(
1431            bbcode_to_carve("[code][b]x[/b][/code]").unwrap(),
1432            "```\n[b]x[/b]\n```\n"
1433        );
1434    }
1435
1436    #[test]
1437    fn plain_carve_syntax_stays_literal() {
1438        assert_eq!(
1439            bbcode_to_carve("literal *stars* and @name").unwrap(),
1440            "literal \\*stars* and \\@name\n"
1441        );
1442    }
1443
1444    #[test]
1445    fn quote_attribution_metadata() {
1446        for (source, caption) in [
1447            ("[quote=\"alice\"]x[/quote]", "^ alice"),
1448            ("[quote='carol']x[/quote]", "^ carol"),
1449            ("[quote=\"Jane Doe\"]x[/quote]", "^ Jane Doe"),
1450            ("[QUOTE=bob;123]x[/QUOTE]", "^ bob #123"),
1451            ("[QUOTE=\"bob;123\"]x[/QUOTE]", "^ bob #123"),
1452            (
1453                "[quote=\"alice\" post_id=12 time=1700000000 user_id=3]x[/quote]",
1454                "^ alice (2023-11-14) #12",
1455            ),
1456            (
1457                "[quote=alice post_id=\"12\" time=\"1700000000\" user_id=\"3\"]x[/quote]",
1458                "^ alice (2023-11-14) #12",
1459            ),
1460            (
1461                "[quote=alice date=\"2024-01-01\" time=\"12\"]x[/quote]",
1462                "^ alice (2024-01-01 12)",
1463            ),
1464            (
1465                "[quote=alice time=170000000]x[/quote]",
1466                "^ alice (1975-05-22)",
1467            ),
1468            (
1469                "[quote=alice time=17000000000]x[/quote]",
1470                "^ alice (2508-09-16)",
1471            ),
1472            ("[quote=\"a]x[/quote]", "^ \"a"),
1473            ("[quote=\"a']x[/quote]", "^ \"a'"),
1474        ] {
1475            let expected = format!("> x\n{caption}\n");
1476            assert_eq!(bbcode_to_carve(source).unwrap(), expected, "{source}");
1477        }
1478    }
1479}