1use crate::utils::calculate_indentation_width_default;
7use crate::utils::is_definition_list_item;
8use crate::utils::mkdocs_attr_list::{ATTR_LIST_PATTERN, is_standalone_attr_list};
9use crate::utils::mkdocs_snippets::is_snippet_block_delimiter;
10use crate::utils::regex_cache::{
11 DISPLAY_MATH_REGEX, EMAIL_PATTERN, EMOJI_SHORTCODE_REGEX, HTML_ENTITY_REGEX, HTML_TAG_PATTERN,
12 HUGO_SHORTCODE_REGEX, INLINE_MATH_REGEX, WIKI_LINK_REGEX,
13};
14use crate::utils::sentence_utils::{
15 get_abbreviations, is_cjk_char, is_cjk_sentence_ending, is_closing_quote, is_opening_quote,
16 text_ends_with_abbreviation,
17};
18use pulldown_cmark::{BrokenLink, CowStr, Event, LinkType, Options, Parser, Tag, TagEnd};
19use std::collections::HashSet;
20use unicode_width::UnicodeWidthStr;
21
22#[derive(Clone, Copy, Debug, Default, PartialEq)]
24pub enum ReflowLengthMode {
25 Chars,
27 #[default]
29 Visual,
30 Bytes,
32}
33
34fn display_len(s: &str, mode: ReflowLengthMode) -> usize {
36 match mode {
37 ReflowLengthMode::Chars => s.chars().count(),
38 ReflowLengthMode::Visual => s.width(),
39 ReflowLengthMode::Bytes => s.len(),
40 }
41}
42
43#[derive(Clone)]
45pub struct ReflowOptions {
46 pub line_length: usize,
48 pub break_on_sentences: bool,
50 pub preserve_breaks: bool,
52 pub sentence_per_line: bool,
54 pub semantic_line_breaks: bool,
56 pub abbreviations: Option<Vec<String>>,
60 pub length_mode: ReflowLengthMode,
62 pub attr_lists: bool,
65 pub myst_roles: bool,
69 pub require_sentence_capital: bool,
74 pub max_list_continuation_indent: Option<usize>,
78 pub defined_references: Option<HashSet<String>>,
92}
93
94impl Default for ReflowOptions {
95 fn default() -> Self {
96 Self {
97 line_length: 80,
98 break_on_sentences: true,
99 preserve_breaks: false,
100 sentence_per_line: false,
101 semantic_line_breaks: false,
102 abbreviations: None,
103 length_mode: ReflowLengthMode::default(),
104 attr_lists: false,
105 myst_roles: false,
106 require_sentence_capital: true,
107 max_list_continuation_indent: None,
108 defined_references: None,
109 }
110 }
111}
112
113pub fn normalize_reference_label(label: &str) -> String {
120 label.split_whitespace().collect::<Vec<_>>().join(" ").to_lowercase()
121}
122
123fn compute_inline_code_mask(text: &str) -> Vec<bool> {
126 let code_spans = extract_code_spans(text);
127 let chars: Vec<char> = text.chars().collect();
128 let mut mask = vec![false; chars.len()];
129 let mut span_it = code_spans.iter().peekable();
130 let mut byte_idx = 0;
131 for (char_idx, ch) in chars.iter().enumerate() {
135 let next_byte_idx = byte_idx + ch.len_utf8();
136 while let Some(span) = span_it.peek() {
137 if span.end <= byte_idx {
138 span_it.next();
139 } else {
140 break;
141 }
142 }
143 if let Some(span) = span_it.peek()
144 && byte_idx >= span.start
145 && byte_idx < span.end
146 {
147 mask[char_idx] = true;
148 }
149 byte_idx = next_byte_idx;
150 }
151 mask
152}
153
154fn footnote_refs_end(chars: &[char], start: usize) -> Option<usize> {
160 let mut pos = start;
161 let mut found = false;
162
163 loop {
164 if chars.get(pos) != Some(&'[') || chars.get(pos + 1) != Some(&'^') {
165 break;
166 }
167 let label_start = pos + 2;
168 let mut label_end = label_start;
169 while matches!(chars.get(label_end), Some(c) if c.is_ascii_alphanumeric() || *c == '_' || *c == '-') {
170 label_end += 1;
171 }
172 if label_end == label_start || chars.get(label_end) != Some(&']') {
173 break;
174 }
175 pos = label_end + 1;
176 found = true;
177 }
178
179 found.then_some(pos)
180}
181
182fn is_sentence_boundary(
186 text: &str,
187 chars: &[char],
188 pos: usize,
189 abbreviations: &HashSet<String>,
190 require_sentence_capital: bool,
191) -> bool {
192 if pos + 1 >= chars.len() {
193 return false;
194 }
195
196 let c = chars[pos];
197 let next_char = chars[pos + 1];
198
199 if is_cjk_sentence_ending(c) {
202 let mut after_punct_pos = pos + 1;
204 while after_punct_pos < chars.len()
205 && (chars[after_punct_pos] == '*' || chars[after_punct_pos] == '_' || chars[after_punct_pos] == '~')
206 {
207 after_punct_pos += 1;
208 }
209
210 while after_punct_pos < chars.len() && chars[after_punct_pos].is_whitespace() {
212 after_punct_pos += 1;
213 }
214
215 if after_punct_pos >= chars.len() {
217 return false;
218 }
219
220 while after_punct_pos < chars.len()
222 && (chars[after_punct_pos] == '*' || chars[after_punct_pos] == '_' || chars[after_punct_pos] == '~')
223 {
224 after_punct_pos += 1;
225 }
226
227 if after_punct_pos >= chars.len() {
228 return false;
229 }
230
231 return true;
234 }
235
236 if c != '.' && c != '!' && c != '?' {
238 return false;
239 }
240
241 let (_space_pos, after_space_pos) = if next_char == ' ' {
243 (pos + 1, pos + 2)
245 } else if is_closing_quote(next_char) && pos + 2 < chars.len() {
246 if chars[pos + 2] == ' ' {
248 (pos + 2, pos + 3)
250 } else if (chars[pos + 2] == '*' || chars[pos + 2] == '_') && pos + 3 < chars.len() && chars[pos + 3] == ' ' {
251 (pos + 3, pos + 4)
253 } else if (chars[pos + 2] == '*' || chars[pos + 2] == '_')
254 && pos + 4 < chars.len()
255 && chars[pos + 3] == chars[pos + 2]
256 && chars[pos + 4] == ' '
257 {
258 (pos + 4, pos + 5)
260 } else {
261 return false;
262 }
263 } else if (next_char == '*' || next_char == '_') && pos + 2 < chars.len() && chars[pos + 2] == ' ' {
264 (pos + 2, pos + 3)
266 } else if (next_char == '*' || next_char == '_')
267 && pos + 3 < chars.len()
268 && chars[pos + 2] == next_char
269 && chars[pos + 3] == ' '
270 {
271 (pos + 3, pos + 4)
273 } else if next_char == '~' && pos + 3 < chars.len() && chars[pos + 2] == '~' && chars[pos + 3] == ' ' {
274 (pos + 3, pos + 4)
276 } else if next_char == '[' {
277 match footnote_refs_end(chars, pos + 1) {
283 Some(end_pos) if chars.get(end_pos) == Some(&' ') => (end_pos, end_pos + 1),
284 _ => return false,
285 }
286 } else {
287 return false;
288 };
289
290 let mut next_char_pos = after_space_pos;
292 while next_char_pos < chars.len() && chars[next_char_pos].is_whitespace() {
293 next_char_pos += 1;
294 }
295
296 if next_char_pos >= chars.len() {
298 return false;
299 }
300
301 let mut first_letter_pos = next_char_pos;
303 while first_letter_pos < chars.len()
304 && (chars[first_letter_pos] == '*'
305 || chars[first_letter_pos] == '_'
306 || chars[first_letter_pos] == '~'
307 || is_opening_quote(chars[first_letter_pos]))
308 {
309 first_letter_pos += 1;
310 }
311
312 if first_letter_pos >= chars.len() {
314 return false;
315 }
316
317 let first_char = chars[first_letter_pos];
318
319 if c == '!' || c == '?' {
321 return true;
322 }
323
324 if pos > 0 {
328 let byte_offset: usize = chars[..=pos].iter().map(|ch| ch.len_utf8()).sum();
330 if text_ends_with_abbreviation(&text[..byte_offset], abbreviations) {
331 return false;
332 }
333
334 if chars[pos - 1].is_numeric() && first_char.is_ascii_digit() {
336 return false;
337 }
338
339 if chars[pos - 1].is_ascii_uppercase() && (pos == 1 || (pos >= 2 && chars[pos - 2].is_whitespace())) {
343 return false;
344 }
345 }
346
347 if require_sentence_capital && !first_char.is_uppercase() && !is_cjk_char(first_char) {
350 return false;
351 }
352
353 true
354}
355
356pub fn split_into_sentences(text: &str) -> Vec<String> {
358 split_into_sentences_custom(text, &None)
359}
360
361pub fn split_into_sentences_custom(text: &str, custom_abbreviations: &Option<Vec<String>>) -> Vec<String> {
363 let abbreviations = get_abbreviations(custom_abbreviations);
364 split_into_sentences_with_set(text, &abbreviations, true)
365}
366
367fn split_into_sentences_with_set(
370 text: &str,
371 abbreviations: &HashSet<String>,
372 require_sentence_capital: bool,
373) -> Vec<String> {
374 let in_code = compute_inline_code_mask(text);
376 let char_vec: Vec<char> = text.chars().collect();
379
380 let mut sentences = Vec::new();
381 let mut current_sentence = String::new();
382 let mut chars = text.chars().peekable();
383 let mut pos = 0;
384
385 while let Some(c) = chars.next() {
386 current_sentence.push(c);
387
388 if !in_code[pos] && is_sentence_boundary(text, &char_vec, pos, abbreviations, require_sentence_capital) {
389 if let Some(end_pos) = footnote_refs_end(&char_vec, pos + 1) {
394 while pos + 1 < end_pos {
395 current_sentence.push(chars.next().unwrap());
396 pos += 1;
397 }
398 }
399
400 while let Some(&next) = chars.peek() {
402 if next == '*' || next == '_' || next == '~' || is_closing_quote(next) {
403 current_sentence.push(chars.next().unwrap());
404 pos += 1;
405 } else {
406 break;
407 }
408 }
409
410 if chars.peek() == Some(&' ') {
412 chars.next();
413 pos += 1;
414 }
415
416 sentences.push(current_sentence.trim().to_string());
417 current_sentence.clear();
418 }
419
420 pos += 1;
421 }
422
423 if !current_sentence.trim().is_empty() {
425 sentences.push(current_sentence.trim().to_string());
426 }
427 sentences
428}
429
430fn is_horizontal_rule(line: &str) -> bool {
432 if line.len() < 3 {
433 return false;
434 }
435
436 let mut chars = line.chars();
439 let Some(first_char) = chars.next() else {
440 return false;
441 };
442 if first_char != '-' && first_char != '_' && first_char != '*' {
443 return false;
444 }
445
446 let mut non_space_count = 1usize; for c in chars {
448 if c == ' ' {
449 continue;
450 }
451 if c != first_char {
452 return false;
453 }
454 non_space_count += 1;
455 }
456 non_space_count >= 3
457}
458
459fn is_numbered_list_item(line: &str) -> bool {
461 let mut chars = line.chars();
462
463 if !chars.next().is_some_and(char::is_numeric) {
465 return false;
466 }
467
468 while let Some(c) = chars.next() {
470 if c == '.' {
471 return chars.next() == Some(' ');
474 }
475 if !c.is_numeric() {
476 return false;
477 }
478 }
479
480 false
481}
482
483fn is_unordered_list_marker(s: &str) -> bool {
485 matches!(s.as_bytes().first(), Some(b'-' | b'*' | b'+'))
486 && !is_horizontal_rule(s)
487 && (s.len() == 1 || s.as_bytes().get(1) == Some(&b' '))
488}
489
490fn is_block_boundary_core(trimmed: &str) -> bool {
493 trimmed.is_empty()
494 || trimmed.starts_with('#')
495 || trimmed.starts_with("```")
496 || trimmed.starts_with("~~~")
497 || trimmed.starts_with('>')
498 || (trimmed.starts_with('[') && trimmed.contains("]:"))
499 || is_horizontal_rule(trimmed)
500 || is_unordered_list_marker(trimmed)
501 || is_numbered_list_item(trimmed)
502 || is_definition_list_item(trimmed)
503 || trimmed.starts_with(":::")
504}
505
506fn is_block_boundary(trimmed: &str) -> bool {
509 is_block_boundary_core(trimmed) || trimmed.starts_with('|')
510}
511
512fn is_paragraph_boundary(trimmed: &str, line: &str) -> bool {
516 is_block_boundary_core(trimmed)
517 || calculate_indentation_width_default(line) >= 4
518 || crate::utils::table_utils::TableUtils::is_potential_table_row(line)
519}
520
521fn has_hard_break(line: &str) -> bool {
527 let line = line.strip_suffix('\r').unwrap_or(line);
528 line.ends_with(" ") || line.ends_with('\\')
529}
530
531fn ends_with_sentence_punct(text: &str) -> bool {
533 text.ends_with('.') || text.ends_with('!') || text.ends_with('?')
534}
535
536fn trim_preserving_hard_break(s: &str) -> String {
542 let s = s.strip_suffix('\r').unwrap_or(s);
544
545 if s.ends_with('\\') {
547 return s.to_string();
549 }
550
551 if s.ends_with(" ") {
553 let content_end = s.trim_end().len();
555 if content_end == 0 {
556 return String::new();
558 }
559 format!("{} ", &s[..content_end])
561 } else {
562 s.trim_end().to_string()
564 }
565}
566
567fn parse_elements(text: &str, options: &ReflowOptions) -> Vec<Element> {
569 parse_markdown_elements_inner(
570 text,
571 options.attr_lists,
572 options.myst_roles,
573 options.defined_references.as_ref(),
574 )
575}
576
577pub fn reflow_line(line: &str, options: &ReflowOptions) -> Vec<String> {
578 if options.sentence_per_line {
580 let elements = parse_elements(line, options);
581 return merge_block_construct_continuations(reflow_elements_sentence_per_line(
582 &elements,
583 &options.abbreviations,
584 options.require_sentence_capital,
585 ));
586 }
587
588 if options.semantic_line_breaks {
590 let elements = parse_elements(line, options);
591 return merge_block_construct_continuations(reflow_elements_semantic(&elements, options));
592 }
593
594 if options.line_length == 0 || display_len(line, options.length_mode) <= options.line_length {
597 return vec![line.to_string()];
598 }
599
600 let elements = parse_elements(line, options);
602
603 merge_block_construct_continuations(reflow_elements(&elements, options))
605}
606
607#[derive(Debug, Clone)]
609enum Element {
610 Text(String),
612 Link(String),
614 ReferenceLink(String),
616 EmptyReferenceLink(String),
618 ShortcutReference(String),
620 InlineImage(String),
622 ReferenceImage(String),
624 EmptyReferenceImage(String),
626 LinkedImage(String),
628 FootnoteReference(String),
630 Strikethrough {
632 content: String,
633 double: bool,
635 },
636 WikiLink(String),
638 InlineMath(String),
640 DisplayMath(String),
642 EmojiShortcode(String),
644 Autolink(String),
646 HtmlTag(String),
648 HtmlEntity(String),
650 HugoShortcode(String),
652 AttrList(String),
654 MystRole(String),
658 Code(String),
660 Bold {
662 content: String,
663 underscore: bool,
665 },
666 Italic {
668 content: String,
669 underscore: bool,
671 },
672}
673
674impl std::fmt::Display for Element {
675 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
676 match self {
677 Element::Text(s) => write!(f, "{s}"),
678 Element::Link(s) => write!(f, "{s}"),
679 Element::ReferenceLink(s) => write!(f, "{s}"),
680 Element::EmptyReferenceLink(s) => write!(f, "{s}"),
681 Element::ShortcutReference(s) => write!(f, "{s}"),
682 Element::InlineImage(s) => write!(f, "{s}"),
683 Element::ReferenceImage(s) => write!(f, "{s}"),
684 Element::EmptyReferenceImage(s) => write!(f, "{s}"),
685 Element::LinkedImage(s) => write!(f, "{s}"),
686 Element::FootnoteReference(s) => write!(f, "{s}"),
687 Element::Strikethrough { content, double } => {
688 let marker = if *double { "~~" } else { "~" };
689 write!(f, "{marker}{content}{marker}")
690 }
691 Element::WikiLink(s) => write!(f, "[[{s}]]"),
692 Element::InlineMath(s) => write!(f, "${s}$"),
693 Element::DisplayMath(s) => write!(f, "$${s}$$"),
694 Element::EmojiShortcode(s) => write!(f, ":{s}:"),
695 Element::Autolink(s) => write!(f, "{s}"),
696 Element::HtmlTag(s) => write!(f, "{s}"),
697 Element::HtmlEntity(s) => write!(f, "{s}"),
698 Element::HugoShortcode(s) => write!(f, "{s}"),
699 Element::AttrList(s) => write!(f, "{s}"),
700 Element::MystRole(s) => write!(f, "{s}"),
701 Element::Code(s) => write!(f, "{s}"),
702 Element::Bold { content, underscore } => {
703 if *underscore {
704 write!(f, "__{content}__")
705 } else {
706 write!(f, "**{content}**")
707 }
708 }
709 Element::Italic { content, underscore } => {
710 if *underscore {
711 write!(f, "_{content}_")
712 } else {
713 write!(f, "*{content}*")
714 }
715 }
716 }
717 }
718}
719
720#[derive(Debug, Clone)]
722struct EmphasisSpan {
723 start: usize,
725 end: usize,
727 content: String,
729 is_strong: bool,
731 is_strikethrough: bool,
733 uses_underscore: bool,
735 strikethrough_double: bool,
738}
739
740fn extract_emphasis_and_code_spans(text: &str) -> (Vec<EmphasisSpan>, Vec<CodeSpan>) {
750 let has_emphasis = text.contains(['*', '_', '~']);
752 let has_code = text.contains('`');
753 if !has_emphasis && !has_code {
754 return (Vec::new(), Vec::new());
755 }
756
757 let mut emphasis_spans = Vec::new();
758 let mut code_spans = Vec::new();
759
760 let mut options = Options::empty();
761 if has_emphasis {
762 options.insert(Options::ENABLE_STRIKETHROUGH);
763 }
764
765 let mut emphasis_stack: Vec<(usize, bool)> = Vec::new(); let mut strong_stack: Vec<(usize, bool)> = Vec::new();
768 let mut strikethrough_stack: Vec<usize> = Vec::new();
769
770 let parser = Parser::new_ext(text, options).into_offset_iter();
771
772 for (event, range) in parser {
773 match event {
774 Event::Code(_) => {
775 code_spans.push(CodeSpan {
776 start: range.start,
777 end: range.end,
778 });
779 }
780 Event::Start(Tag::Emphasis) => {
781 let uses_underscore = text.get(range.start..range.start + 1) == Some("_");
783 emphasis_stack.push((range.start, uses_underscore));
784 }
785 Event::End(TagEnd::Emphasis) => {
786 if let Some((start_byte, uses_underscore)) = emphasis_stack.pop() {
787 let content_start = start_byte + 1;
788 let content_end = range.end - 1;
789 if content_end > content_start
790 && let Some(content) = text.get(content_start..content_end)
791 {
792 emphasis_spans.push(EmphasisSpan {
793 start: start_byte,
794 end: range.end,
795 content: content.to_string(),
796 is_strong: false,
797 is_strikethrough: false,
798 uses_underscore,
799 strikethrough_double: false,
800 });
801 }
802 }
803 }
804 Event::Start(Tag::Strong) => {
805 let uses_underscore = text.get(range.start..range.start + 2) == Some("__");
806 strong_stack.push((range.start, uses_underscore));
807 }
808 Event::End(TagEnd::Strong) => {
809 if let Some((start_byte, uses_underscore)) = strong_stack.pop() {
810 let content_start = start_byte + 2;
811 let content_end = range.end - 2;
812 if content_end > content_start
813 && let Some(content) = text.get(content_start..content_end)
814 {
815 emphasis_spans.push(EmphasisSpan {
816 start: start_byte,
817 end: range.end,
818 content: content.to_string(),
819 is_strong: true,
820 is_strikethrough: false,
821 uses_underscore,
822 strikethrough_double: false,
823 });
824 }
825 }
826 }
827 Event::Start(Tag::Strikethrough) => {
828 strikethrough_stack.push(range.start);
829 }
830 Event::End(TagEnd::Strikethrough) => {
831 if let Some(start_byte) = strikethrough_stack.pop() {
832 let double = text.get(start_byte..start_byte + 2) == Some("~~");
833 let marker_len = if double { 2 } else { 1 };
834 let content_start = start_byte + marker_len;
835 let content_end = range.end - marker_len;
836 if content_end > content_start
837 && let Some(content) = text.get(content_start..content_end)
838 {
839 emphasis_spans.push(EmphasisSpan {
840 start: start_byte,
841 end: range.end,
842 content: content.to_string(),
843 is_strong: false,
844 is_strikethrough: true,
845 uses_underscore: false,
846 strikethrough_double: double,
847 });
848 }
849 }
850 }
851 _ => {}
852 }
853 }
854
855 emphasis_spans.sort_by_key(|s| s.start);
856 (emphasis_spans, code_spans)
857}
858
859#[derive(Debug, Clone)]
860struct CodeSpan {
861 start: usize,
862 end: usize,
863}
864
865fn extract_code_spans(text: &str) -> Vec<CodeSpan> {
866 if !text.contains('`') {
868 return Vec::new();
869 }
870
871 let mut spans = Vec::new();
872 let parser = Parser::new(text).into_offset_iter();
873 for (event, range) in parser {
874 if let Event::Code(_) = event {
875 spans.push(CodeSpan {
876 start: range.start,
877 end: range.end,
878 });
879 }
880 }
881 spans
882}
883
884#[derive(Debug, Clone)]
885struct LinkSpan {
886 start: usize,
887 end: usize,
888 link_type: Option<LinkType>,
889 is_image: bool,
890 is_footnote: bool,
891}
892
893fn extract_link_spans(text: &str, defined_references: Option<&HashSet<String>>) -> Vec<LinkSpan> {
894 if !text.contains('[') {
897 return Vec::new();
898 }
899
900 let mut spans = Vec::new();
901 let mut options = Options::empty();
902 options.insert(Options::ENABLE_FOOTNOTES);
903
904 let resolve = move |link: BrokenLink<'_>| -> Option<(CowStr<'_>, CowStr<'_>)> {
921 let atomic = match link.link_type {
926 LinkType::Shortcut | LinkType::ShortcutUnknown => match defined_references {
927 Some(defs) => defs.contains(&normalize_reference_label(link.reference.as_ref())),
928 None => true,
929 },
930 _ => true,
931 };
932 atomic.then_some((CowStr::Borrowed(""), CowStr::Borrowed("")))
933 };
934 let parser = Parser::new_with_broken_link_callback(text, options, Some(resolve)).into_offset_iter();
935 let mut stack = Vec::new();
936
937 for (event, range) in parser {
938 match event {
939 Event::Start(Tag::Link { link_type, .. }) => {
940 stack.push((range.start, Some(link_type), false));
941 }
942 Event::Start(Tag::Image { link_type, .. }) => {
943 stack.push((range.start, Some(link_type), true));
944 }
945 Event::End(TagEnd::Link) => {
946 if let Some((start_byte, link_type, is_image)) = stack.pop()
947 && stack.is_empty()
948 {
949 spans.push(LinkSpan {
950 start: start_byte,
951 end: range.end,
952 link_type,
953 is_image,
954 is_footnote: false,
955 });
956 }
957 }
958 Event::End(TagEnd::Image) => {
959 if let Some((start_byte, link_type, is_image)) = stack.pop()
960 && stack.is_empty()
961 {
962 spans.push(LinkSpan {
963 start: start_byte,
964 end: range.end,
965 link_type,
966 is_image,
967 is_footnote: false,
968 });
969 }
970 }
971 Event::FootnoteReference(_) if stack.is_empty() => {
972 spans.push(LinkSpan {
973 start: range.start,
974 end: range.end,
975 link_type: None,
976 is_image: false,
977 is_footnote: true,
978 });
979 }
980 _ => {}
981 }
982 }
983
984 spans.sort_by_key(|s| s.start);
985 spans
986}
987
988fn myst_role_len_at(text: &str, absolute_pos: usize, code_spans: &[CodeSpan]) -> Option<usize> {
996 let bytes = text.as_bytes();
997 if bytes.first() != Some(&b'{') {
998 return None;
999 }
1000
1001 let mut j = 1;
1003 match bytes.get(j) {
1004 Some(&b) if b.is_ascii_alphabetic() || b == b'_' => {}
1005 _ => return None,
1006 }
1007 while let Some(&b) = bytes.get(j) {
1008 if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b':' | b'.') {
1009 j += 1;
1010 } else {
1011 break;
1012 }
1013 }
1014 if bytes.get(j) != Some(&b'}') {
1015 return None;
1016 }
1017 j += 1; let code_span_start = absolute_pos + j;
1021 if let Ok(idx) = code_spans.binary_search_by_key(&code_span_start, |span| span.start) {
1022 let span = &code_spans[idx];
1023 let code_span_len = span.end - span.start;
1024 return Some(j + code_span_len);
1025 }
1026
1027 None
1028}
1029
1030fn inline_math_len_at_start(s: &str) -> Option<usize> {
1037 let bytes = s.as_bytes();
1038 if bytes.first() != Some(&b'$') || bytes.get(1) == Some(&b'$') {
1040 return None;
1041 }
1042 let close = 1 + s[1..].find('$')?;
1045 if bytes.get(close + 1) == Some(&b'$') {
1047 return None;
1048 }
1049 Some(close + 1)
1050}
1051
1052#[derive(Clone, Copy, Debug)]
1054struct PatternMatch {
1055 start: usize,
1056 end: usize,
1057}
1058
1059#[derive(Clone, Copy)]
1073enum PatternCache {
1074 Unsearched,
1075 NotFound,
1076 Found(PatternMatch),
1077}
1078
1079impl PatternCache {
1080 fn earliest_in(
1084 &mut self,
1085 remaining: &str,
1086 cursor: usize,
1087 find: impl FnOnce(&str) -> Option<(usize, usize)>,
1088 ) -> Option<(usize, usize)> {
1089 let stale = match self {
1090 PatternCache::Found(pm) => pm.start < cursor,
1091 PatternCache::NotFound => false,
1092 PatternCache::Unsearched => true,
1093 };
1094 if stale {
1095 *self = match find(remaining) {
1096 Some((start, end)) => PatternCache::Found(PatternMatch {
1097 start: cursor + start,
1098 end: cursor + end,
1099 }),
1100 None => PatternCache::NotFound,
1101 };
1102 }
1103 match self {
1104 PatternCache::Found(pm) => Some((pm.start - cursor, pm.end - cursor)),
1105 _ => None,
1106 }
1107 }
1108}
1109
1110fn parse_markdown_elements_inner(
1121 text: &str,
1122 attr_lists: bool,
1123 myst_roles: bool,
1124 defined_references: Option<&HashSet<String>>,
1125) -> Vec<Element> {
1126 let mut elements = Vec::new();
1127 let mut remaining = text;
1128
1129 let (emphasis_spans, code_spans) = extract_emphasis_and_code_spans(text);
1134 let link_spans = extract_link_spans(text, defined_references);
1135
1136 let mut cached_wiki_link = PatternCache::Unsearched;
1139 let mut cached_display_math = PatternCache::Unsearched;
1140 let mut cached_inline_math = PatternCache::Unsearched;
1141 let mut cached_emoji = PatternCache::Unsearched;
1142 let mut cached_html_entity = PatternCache::Unsearched;
1143 let mut cached_hugo_shortcode = PatternCache::Unsearched;
1144 let mut cached_html_tag = PatternCache::Unsearched;
1145 let mut cached_next_curly = PatternCache::Unsearched;
1146
1147 let mut link_span_idx = 0usize;
1151 let mut emphasis_span_idx = 0usize;
1152 let mut code_span_idx = 0usize;
1153
1154 while !remaining.is_empty() {
1155 let current_offset = text.len() - remaining.len();
1157 let mut earliest_match: Option<(usize, usize, &str)> = None;
1160
1161 while link_span_idx < link_spans.len() && link_spans[link_span_idx].start < current_offset {
1163 link_span_idx += 1;
1164 }
1165 let next_link: Option<&LinkSpan> = link_spans.get(link_span_idx);
1166
1167 if let Some(span) = next_link {
1168 let pos_in_remaining = span.start - current_offset;
1169 if earliest_match
1170 .as_ref()
1171 .is_none_or(|(start, _, _)| pos_in_remaining < *start)
1172 {
1173 let match_end = span.end - current_offset;
1174 earliest_match = Some((pos_in_remaining, match_end, "link_span"));
1175 }
1176 }
1177
1178 if let Some((start, end)) = cached_wiki_link.earliest_in(remaining, current_offset, |suffix| {
1180 WIKI_LINK_REGEX.find(suffix).map(|m| (m.start(), m.end()))
1181 }) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
1182 {
1183 earliest_match = Some((start, end, "wiki_link"));
1184 }
1185
1186 if let Some((start, end)) = cached_display_math.earliest_in(remaining, current_offset, |suffix| {
1188 DISPLAY_MATH_REGEX.find(suffix).map(|m| (m.start(), m.end()))
1189 }) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
1190 {
1191 earliest_match = Some((start, end, "display_math"));
1192 }
1193
1194 let inline_math_probe = if current_offset > 0 && text.as_bytes()[current_offset - 1] == b'$' {
1208 inline_math_len_at_start(remaining).map(|len| (0, len))
1209 } else {
1210 None
1211 };
1212 if let Some((start, end)) = inline_math_probe.or_else(|| {
1213 cached_inline_math.earliest_in(remaining, current_offset, |suffix| {
1214 INLINE_MATH_REGEX
1215 .find(suffix)
1216 .ok()
1217 .flatten()
1218 .map(|m| (m.start(), m.end()))
1219 })
1220 }) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
1221 {
1222 earliest_match = Some((start, end, "inline_math"));
1223 }
1224
1225 if let Some((start, end)) = cached_emoji.earliest_in(remaining, current_offset, |suffix| {
1227 EMOJI_SHORTCODE_REGEX.find(suffix).map(|m| (m.start(), m.end()))
1228 }) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
1229 {
1230 earliest_match = Some((start, end, "emoji"));
1231 }
1232
1233 if let Some((start, end)) = cached_html_entity.earliest_in(remaining, current_offset, |suffix| {
1235 HTML_ENTITY_REGEX.find(suffix).map(|m| (m.start(), m.end()))
1236 }) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
1237 {
1238 earliest_match = Some((start, end, "html_entity"));
1239 }
1240
1241 if let Some((start, end)) = cached_hugo_shortcode.earliest_in(remaining, current_offset, |suffix| {
1244 HUGO_SHORTCODE_REGEX.find(suffix).map(|m| (m.start(), m.end()))
1245 }) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
1246 {
1247 earliest_match = Some((start, end, "hugo_shortcode"));
1248 }
1249
1250 if let Some((start, end)) = cached_html_tag.earliest_in(remaining, current_offset, |suffix| {
1257 let mut from = 0;
1258 while let Some(m) = HTML_TAG_PATTERN.find(&suffix[from..]) {
1259 let (tag_start, tag_end) = (from + m.start(), from + m.end());
1260 let tag = &suffix[tag_start..tag_end];
1261 let is_url_autolink = tag.starts_with("<http://")
1263 || tag.starts_with("<https://")
1264 || tag.starts_with("<mailto:")
1265 || tag.starts_with("<ftp://")
1266 || tag.starts_with("<ftps://");
1267 let is_email_autolink = {
1270 let content = tag.trim_start_matches('<').trim_end_matches('>');
1271 EMAIL_PATTERN.is_match(content)
1272 };
1273 if is_url_autolink || is_email_autolink {
1274 from = tag_end;
1275 } else {
1276 return Some((tag_start, tag_end));
1277 }
1278 }
1279 None
1280 }) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
1281 {
1282 earliest_match = Some((start, end, "html_tag"));
1283 }
1284
1285 let mut next_special = remaining.len();
1287 let mut special_type = "";
1288 let mut pulldown_emphasis: Option<&EmphasisSpan> = None;
1289 let mut attr_list_len: usize = 0;
1290 let mut myst_role_len: usize = 0;
1291
1292 while code_span_idx < code_spans.len() && code_spans[code_span_idx].start < current_offset {
1294 code_span_idx += 1;
1295 }
1296 let next_code_span: Option<&CodeSpan> = code_spans.get(code_span_idx);
1297 if let Some(span) = next_code_span {
1298 let pos_in_remaining = span.start - current_offset;
1299 if pos_in_remaining < next_special {
1300 next_special = pos_in_remaining;
1301 special_type = "pulldown_code";
1302 }
1303 }
1304
1305 let next_curly_pos = cached_next_curly
1308 .earliest_in(remaining, current_offset, |suffix| {
1309 suffix.find('{').map(|pos| (pos, pos + 1))
1310 })
1311 .map(|(start, _)| start);
1312
1313 if myst_roles
1318 && let Some(pos) = next_curly_pos
1319 && pos < next_special
1320 && let Some(role_len) = myst_role_len_at(&remaining[pos..], current_offset + pos, &code_spans)
1321 {
1322 next_special = pos;
1323 special_type = "myst_role";
1324 myst_role_len = role_len;
1325 }
1326
1327 if attr_lists
1329 && let Some(pos) = next_curly_pos
1330 && pos < next_special
1331 && let Some(m) = ATTR_LIST_PATTERN.find(&remaining[pos..])
1332 && m.start() == 0
1333 {
1334 next_special = pos;
1335 special_type = "attr_list";
1336 attr_list_len = m.end();
1337 }
1338
1339 while emphasis_span_idx < emphasis_spans.len() && emphasis_spans[emphasis_span_idx].start < current_offset {
1341 emphasis_span_idx += 1;
1342 }
1343 if let Some(span) = emphasis_spans.get(emphasis_span_idx) {
1344 let pos_in_remaining = span.start - current_offset;
1345 if pos_in_remaining < next_special {
1346 next_special = pos_in_remaining;
1347 special_type = "pulldown_emphasis";
1348 pulldown_emphasis = Some(span);
1349 }
1350 }
1351
1352 let should_process_markdown_link = if let Some((pos, _, _)) = earliest_match {
1354 pos < next_special
1355 } else {
1356 false
1357 };
1358
1359 if should_process_markdown_link {
1360 let (pos, match_end, pattern_type) = earliest_match.unwrap();
1361
1362 if pos > 0 {
1364 elements.push(Element::Text(remaining[..pos].to_string()));
1365 }
1366
1367 match pattern_type {
1369 "link_span" => {
1370 let span = next_link.unwrap();
1371 let raw_text = remaining[pos..match_end].to_string();
1372 if span.is_footnote {
1373 elements.push(Element::FootnoteReference(raw_text));
1374 } else if span.is_image {
1375 match span.link_type {
1376 Some(LinkType::Inline) => elements.push(Element::InlineImage(raw_text)),
1377 Some(LinkType::Reference)
1380 | Some(LinkType::ReferenceUnknown)
1381 | Some(LinkType::Shortcut)
1382 | Some(LinkType::ShortcutUnknown) => elements.push(Element::ReferenceImage(raw_text)),
1383 Some(LinkType::Collapsed) | Some(LinkType::CollapsedUnknown) => {
1384 elements.push(Element::EmptyReferenceImage(raw_text))
1385 }
1386 _ => elements.push(Element::InlineImage(raw_text)),
1387 }
1388 } else {
1389 match span.link_type {
1390 Some(LinkType::Inline) => {
1391 if raw_text.starts_with('[') && raw_text.contains("![") {
1392 elements.push(Element::LinkedImage(raw_text));
1393 } else {
1394 elements.push(Element::Link(raw_text));
1395 }
1396 }
1397 Some(LinkType::Reference) | Some(LinkType::ReferenceUnknown) => {
1400 elements.push(Element::ReferenceLink(raw_text))
1401 }
1402 Some(LinkType::Collapsed) | Some(LinkType::CollapsedUnknown) => {
1403 elements.push(Element::EmptyReferenceLink(raw_text))
1404 }
1405 Some(LinkType::Shortcut) | Some(LinkType::ShortcutUnknown) => {
1406 elements.push(Element::ShortcutReference(raw_text))
1407 }
1408 Some(LinkType::Autolink) | Some(LinkType::Email) => {
1409 elements.push(Element::Autolink(raw_text))
1410 }
1411 _ => elements.push(Element::Link(raw_text)),
1412 }
1413 }
1414 remaining = &remaining[match_end..];
1415 }
1416 "wiki_link" => {
1417 if let Some(caps) = WIKI_LINK_REGEX.captures(remaining) {
1418 let content = caps.get(1).map_or("", |m| m.as_str());
1419 elements.push(Element::WikiLink(content.to_string()));
1420 remaining = &remaining[match_end..];
1421 } else {
1422 elements.push(Element::Text("[[".to_string()));
1423 remaining = &remaining[2..];
1424 }
1425 }
1426 "display_math" => {
1427 if let Some(caps) = DISPLAY_MATH_REGEX.captures(remaining) {
1428 let math = caps.get(1).map_or("", |m| m.as_str());
1429 elements.push(Element::DisplayMath(math.to_string()));
1430 remaining = &remaining[match_end..];
1431 } else {
1432 elements.push(Element::Text("$$".to_string()));
1433 remaining = &remaining[2..];
1434 }
1435 }
1436 "inline_math" => {
1437 if let Ok(Some(caps)) = INLINE_MATH_REGEX.captures(remaining) {
1438 let math = caps.get(1).map_or("", |m| m.as_str());
1439 elements.push(Element::InlineMath(math.to_string()));
1440 remaining = &remaining[match_end..];
1441 } else {
1442 elements.push(Element::Text("$".to_string()));
1443 remaining = &remaining[1..];
1444 }
1445 }
1446 "emoji" => {
1447 if let Some(caps) = EMOJI_SHORTCODE_REGEX.captures(remaining) {
1448 let emoji = caps.get(1).map_or("", |m| m.as_str());
1449 elements.push(Element::EmojiShortcode(emoji.to_string()));
1450 remaining = &remaining[match_end..];
1451 } else {
1452 elements.push(Element::Text(":".to_string()));
1453 remaining = &remaining[1..];
1454 }
1455 }
1456 "html_entity" => {
1457 elements.push(Element::HtmlEntity(remaining[pos..match_end].to_string()));
1459 remaining = &remaining[match_end..];
1460 }
1461 "hugo_shortcode" => {
1462 elements.push(Element::HugoShortcode(remaining[pos..match_end].to_string()));
1464 remaining = &remaining[match_end..];
1465 }
1466 "html_tag" => {
1467 elements.push(Element::HtmlTag(remaining[pos..match_end].to_string()));
1469 remaining = &remaining[match_end..];
1470 }
1471 _ => unreachable!("unknown pattern type: {}", pattern_type),
1472 }
1473 } else {
1474 if next_special > 0 && next_special < remaining.len() {
1478 elements.push(Element::Text(remaining[..next_special].to_string()));
1479 remaining = &remaining[next_special..];
1480 }
1481
1482 match special_type {
1484 "pulldown_code" => {
1485 let span = next_code_span.unwrap();
1486 let span_len = span.end - span.start;
1487 let code = &remaining[..span_len];
1488 elements.push(Element::Code(code.to_string()));
1489 remaining = &remaining[span_len..];
1490 }
1491 "attr_list" => {
1492 elements.push(Element::AttrList(remaining[..attr_list_len].to_string()));
1493 remaining = &remaining[attr_list_len..];
1494 }
1495 "myst_role" => {
1496 elements.push(Element::MystRole(remaining[..myst_role_len].to_string()));
1497 remaining = &remaining[myst_role_len..];
1498 }
1499 "pulldown_emphasis" => {
1500 let span = pulldown_emphasis.expect("pulldown_emphasis must be set");
1502 let span_len = span.end - span.start;
1503 if span.is_strikethrough {
1504 elements.push(Element::Strikethrough {
1505 content: span.content.clone(),
1506 double: span.strikethrough_double,
1507 });
1508 } else if span.is_strong {
1509 elements.push(Element::Bold {
1510 content: span.content.clone(),
1511 underscore: span.uses_underscore,
1512 });
1513 } else {
1514 elements.push(Element::Italic {
1515 content: span.content.clone(),
1516 underscore: span.uses_underscore,
1517 });
1518 }
1519 remaining = &remaining[span_len..];
1520 }
1521 _ => {
1522 elements.push(Element::Text(remaining.to_string()));
1524 break;
1525 }
1526 }
1527 }
1528 }
1529
1530 let mut merged_elements = Vec::new();
1532 for el in elements {
1533 match el {
1534 Element::Text(s) => {
1535 if let Some(Element::Text(last_s)) = merged_elements.last_mut() {
1536 last_s.push_str(&s);
1537 } else {
1538 merged_elements.push(Element::Text(s));
1539 }
1540 }
1541 other => merged_elements.push(other),
1542 }
1543 }
1544 merged_elements
1545}
1546
1547fn should_insert_space_before_join(current: &str) -> bool {
1548 !current.is_empty()
1549 && !current.ends_with(' ')
1550 && !current.ends_with('(')
1551 && !current.ends_with('[')
1552 && !current.ends_with('-')
1553}
1554
1555fn is_setext_or_thematic(text: &str) -> bool {
1561 let mut marker = '\0';
1562 let mut count = 0usize;
1563 let mut has_space = false;
1564 for c in text.chars() {
1565 match c {
1566 ' ' | '\t' => has_space = true,
1567 '-' | '=' | '*' | '_' => {
1568 if marker == '\0' {
1569 marker = c;
1570 } else if c != marker {
1571 return false;
1572 }
1573 count += 1;
1574 }
1575 _ => return false,
1576 }
1577 }
1578 match marker {
1579 '=' => !has_space,
1580 '-' => !has_space || count >= 3,
1581 '*' | '_' => count >= 3,
1582 _ => false,
1583 }
1584}
1585
1586fn starts_block_construct(text: &str) -> bool {
1598 let text = text.trim_start();
1599 let bytes = text.as_bytes();
1600 let Some(&first) = bytes.first() else {
1601 return false;
1602 };
1603 let marker_then_boundary = |len: usize| bytes.len() == len || bytes[len] == b' ' || bytes[len] == b'\t';
1604 match first {
1605 b'>' => true,
1607 b'-' | b'*' | b'+' => marker_then_boundary(1) || is_setext_or_thematic(text),
1608 b'_' | b'=' => is_setext_or_thematic(text),
1609 b'#' => {
1610 let hashes = bytes.iter().take_while(|&&b| b == b'#').count();
1611 hashes <= 6 && marker_then_boundary(hashes)
1612 }
1613 b'`' => bytes.iter().take_while(|&&b| b == b'`').count() >= 3,
1614 b'~' => bytes.iter().take_while(|&&b| b == b'~').count() >= 3,
1615 b'0'..=b'9' => {
1616 let digits = bytes.iter().take_while(|b| b.is_ascii_digit()).count();
1617 digits <= 9
1618 && bytes.len() > digits
1619 && (bytes[digits] == b'.' || bytes[digits] == b')')
1620 && marker_then_boundary(digits + 1)
1621 }
1622 b'[' => {
1630 let mut escaped = false;
1631 let mut label_close = None;
1632 for (i, &b) in bytes.iter().enumerate().skip(1) {
1633 if escaped {
1634 escaped = false;
1635 } else if b == b'\\' {
1636 escaped = true;
1637 } else if b == b']' {
1638 label_close = Some(i);
1639 break;
1640 }
1641 }
1642 label_close.is_some_and(|i| bytes.get(i + 1) == Some(&b':'))
1643 }
1644 b'<' => crate::utils::html_block::parse_html_block_start(text).is_some(),
1647 _ => false,
1648 }
1649}
1650
1651fn merge_block_construct_continuations(lines: Vec<String>) -> Vec<String> {
1660 let mut merged: Vec<String> = Vec::with_capacity(lines.len());
1661 for line in lines {
1662 match merged.last_mut() {
1663 Some(prev) if starts_block_construct(&line) => {
1664 prev.push(' ');
1665 prev.push_str(line.trim_start());
1666 }
1667 _ => merged.push(line),
1668 }
1669 }
1670 merged
1671}
1672
1673fn reflow_elements_sentence_per_line(
1675 elements: &[Element],
1676 custom_abbreviations: &Option<Vec<String>>,
1677 require_sentence_capital: bool,
1678) -> Vec<String> {
1679 let abbreviations = get_abbreviations(custom_abbreviations);
1680 let mut lines = Vec::new();
1681 let mut current_line = String::new();
1682
1683 for (idx, element) in elements.iter().enumerate() {
1684 let element_str = format!("{element}");
1685
1686 if let Element::Text(text) = element {
1688 let combined = format!("{current_line}{text}");
1690 let sentences = split_into_sentences_with_set(&combined, &abbreviations, require_sentence_capital);
1692
1693 if sentences.len() > 1 {
1694 for (i, sentence) in sentences.iter().enumerate() {
1696 if i == 0 {
1697 let trimmed = sentence.trim();
1700
1701 if text_ends_with_abbreviation(trimmed, &abbreviations) {
1702 current_line.clone_from(sentence);
1704 } else {
1705 lines.push(sentence.clone());
1707 current_line.clear();
1708 }
1709 } else if i == sentences.len() - 1 {
1710 let trimmed = sentence.trim();
1712 let ends_with_sentence_punct = ends_with_sentence_punct(trimmed);
1713
1714 if ends_with_sentence_punct && !text_ends_with_abbreviation(trimmed, &abbreviations) {
1715 lines.push(sentence.clone());
1717 current_line.clear();
1718 } else {
1719 current_line.clone_from(sentence);
1721 }
1722 } else {
1723 lines.push(sentence.clone());
1725 }
1726 }
1727 } else {
1728 let trimmed = combined.trim();
1730
1731 if trimmed.is_empty() {
1735 continue;
1736 }
1737
1738 let ends_with_sentence_punct = ends_with_sentence_punct(trimmed);
1739
1740 if ends_with_sentence_punct && !text_ends_with_abbreviation(trimmed, &abbreviations) {
1741 lines.push(trimmed.to_string());
1743 current_line.clear();
1744 } else {
1745 current_line = combined;
1747 }
1748 }
1749 } else if let Element::Italic { content, underscore } = element {
1750 let marker = if *underscore { "_" } else { "*" };
1752 handle_emphasis_sentence_split(
1753 content,
1754 marker,
1755 &abbreviations,
1756 require_sentence_capital,
1757 &mut current_line,
1758 &mut lines,
1759 );
1760 } else if let Element::Bold { content, underscore } = element {
1761 let marker = if *underscore { "__" } else { "**" };
1763 handle_emphasis_sentence_split(
1764 content,
1765 marker,
1766 &abbreviations,
1767 require_sentence_capital,
1768 &mut current_line,
1769 &mut lines,
1770 );
1771 } else if let Element::Strikethrough { content, double } = element {
1772 handle_emphasis_sentence_split(
1774 content,
1775 if *double { "~~" } else { "~" },
1776 &abbreviations,
1777 require_sentence_capital,
1778 &mut current_line,
1779 &mut lines,
1780 );
1781 } else {
1782 let is_adjacent = if idx > 0 {
1785 match &elements[idx - 1] {
1786 Element::Text(t) => !t.is_empty() && !t.ends_with(char::is_whitespace),
1787 _ => true,
1788 }
1789 } else {
1790 false
1791 };
1792
1793 if !is_adjacent && should_insert_space_before_join(¤t_line) {
1795 current_line.push(' ');
1796 }
1797 current_line.push_str(&element_str);
1798 }
1799 }
1800
1801 if !current_line.is_empty() {
1803 lines.push(current_line.trim().to_string());
1804 }
1805 lines
1806}
1807
1808fn handle_emphasis_sentence_split(
1810 content: &str,
1811 marker: &str,
1812 abbreviations: &HashSet<String>,
1813 require_sentence_capital: bool,
1814 current_line: &mut String,
1815 lines: &mut Vec<String>,
1816) {
1817 let sentences = split_into_sentences_with_set(content, abbreviations, require_sentence_capital);
1819
1820 if sentences.len() <= 1 {
1821 if should_insert_space_before_join(current_line) {
1823 current_line.push(' ');
1824 }
1825 current_line.push_str(marker);
1826 current_line.push_str(content);
1827 current_line.push_str(marker);
1828
1829 let trimmed = content.trim();
1831 let ends_with_punct = ends_with_sentence_punct(trimmed);
1832 if ends_with_punct && !text_ends_with_abbreviation(trimmed, abbreviations) {
1833 lines.push(current_line.clone());
1834 current_line.clear();
1835 }
1836 } else {
1837 for (i, sentence) in sentences.iter().enumerate() {
1839 let trimmed = sentence.trim();
1840 if trimmed.is_empty() {
1841 continue;
1842 }
1843
1844 if i == 0 {
1845 if should_insert_space_before_join(current_line) {
1847 current_line.push(' ');
1848 }
1849 current_line.push_str(marker);
1850 current_line.push_str(trimmed);
1851 current_line.push_str(marker);
1852
1853 let ends_with_punct = ends_with_sentence_punct(trimmed);
1855 if ends_with_punct && !text_ends_with_abbreviation(trimmed, abbreviations) {
1856 lines.push(current_line.clone());
1857 current_line.clear();
1858 }
1859 } else if i == sentences.len() - 1 {
1860 let ends_with_punct = ends_with_sentence_punct(trimmed);
1862
1863 let mut line = String::new();
1864 line.push_str(marker);
1865 line.push_str(trimmed);
1866 line.push_str(marker);
1867
1868 if ends_with_punct && !text_ends_with_abbreviation(trimmed, abbreviations) {
1869 lines.push(line);
1870 } else {
1871 *current_line = line;
1873 }
1874 } else {
1875 let mut line = String::new();
1877 line.push_str(marker);
1878 line.push_str(trimmed);
1879 line.push_str(marker);
1880 lines.push(line);
1881 }
1882 }
1883 }
1884}
1885
1886const BREAK_WORDS: &[&str] = &[
1890 "and",
1891 "or",
1892 "but",
1893 "nor",
1894 "yet",
1895 "so",
1896 "for",
1897 "which",
1898 "that",
1899 "because",
1900 "when",
1901 "if",
1902 "while",
1903 "where",
1904 "although",
1905 "though",
1906 "unless",
1907 "since",
1908 "after",
1909 "before",
1910 "until",
1911 "as",
1912 "once",
1913 "whether",
1914 "however",
1915 "therefore",
1916 "moreover",
1917 "furthermore",
1918 "nevertheless",
1919 "whereas",
1920];
1921
1922fn is_clause_punctuation(c: char) -> bool {
1924 matches!(c, ',' | ';' | ':' | '\u{2014}') }
1926
1927fn clause_break_allowed_after(chars: &[char], i: usize) -> bool {
1935 if chars[i] == '\u{2014}' {
1936 return true;
1937 }
1938 match chars.get(i + 1) {
1939 None => true,
1940 Some(next) => next.is_whitespace(),
1941 }
1942}
1943
1944fn paren_group_end<'a>(slice: &'a str, element_spans: &[(usize, usize)], offset: usize) -> Option<(usize, &'a str)> {
1958 debug_assert!(slice.starts_with('('));
1959 let mut depth: i32 = 0;
1960 for (local_byte, c) in slice.char_indices() {
1961 let global_byte = offset + local_byte;
1962 if depth > 0 && is_inside_element(global_byte, element_spans) {
1967 continue;
1968 }
1969 match c {
1970 '(' => depth += 1,
1971 ')' => {
1972 depth -= 1;
1973 if depth == 0 {
1974 let end = local_byte + 1;
1975 let inner = &slice[1..local_byte];
1976 return Some((end, inner));
1977 }
1978 }
1979 _ => {}
1980 }
1981 }
1982 None
1983}
1984
1985fn split_at_parenthetical(
2002 text: &str,
2003 line_length: usize,
2004 element_spans: &[(usize, usize)],
2005 length_mode: ReflowLengthMode,
2006) -> Option<(String, String)> {
2007 let min_first_len = ((line_length as f64) * MIN_SPLIT_RATIO) as usize;
2008
2009 if text.starts_with('(')
2011 && let Some((end_local, inner)) = paren_group_end(text, element_spans, 0)
2012 && inner.contains(' ')
2013 {
2014 let tail = &text[end_local..];
2018 let attached_len = tail
2019 .char_indices()
2020 .take_while(|(_, c)| is_closing_quote(*c) || is_clause_punctuation(*c))
2021 .last()
2022 .map_or(0, |(idx, c)| idx + c.len_utf8());
2023 let first_end = end_local + attached_len;
2024 let rest_start = first_end;
2025 let first = &text[..first_end];
2026 let first_len = display_len(first, length_mode);
2027 if first_len <= line_length {
2030 let rest = text[rest_start..].trim_start();
2031 if !rest.is_empty() {
2032 return Some((first.to_string(), rest.to_string()));
2033 }
2034 }
2035 }
2036
2037 let mut best_open_byte: Option<usize> = None;
2039 let mut pos = 0usize;
2040 while pos < text.len() {
2041 if text.as_bytes()[pos] != b'(' {
2043 let c = text[pos..].chars().next().unwrap();
2044 pos += c.len_utf8();
2045 continue;
2046 }
2047 if is_inside_element(pos, element_spans) {
2049 pos += 1;
2050 continue;
2051 }
2052 if let Some((end_local, inner)) = paren_group_end(&text[pos..], element_spans, pos) {
2053 let first = text[..pos].trim_end();
2054 let first_len = display_len(first, length_mode);
2055 if !first.is_empty()
2056 && first_len >= min_first_len
2057 && first_len <= line_length
2058 && inner.contains(' ')
2059 && best_open_byte.is_none_or(|prev| pos > prev)
2060 {
2061 best_open_byte = Some(pos);
2062 }
2063 pos += end_local;
2064 } else {
2065 pos += 1;
2066 }
2067 }
2068
2069 let open_byte = best_open_byte?;
2070 let first = text[..open_byte].trim_end().to_string();
2071 let rest = text[open_byte..].to_string();
2072 if first.is_empty() || rest.trim().is_empty() {
2073 return None;
2074 }
2075 Some((first, rest))
2076}
2077
2078fn compute_element_spans(elements: &[Element]) -> Vec<(usize, usize)> {
2082 let mut spans = Vec::new();
2083 let mut offset = 0;
2084 for element in elements {
2085 let rendered = format!("{element}");
2086 let len = rendered.len();
2087 if !matches!(element, Element::Text(_)) {
2088 spans.push((offset, offset + len));
2089 }
2090 offset += len;
2091 }
2092 spans
2093}
2094
2095fn is_inside_element(pos: usize, spans: &[(usize, usize)]) -> bool {
2097 spans.iter().any(|(start, end)| pos > *start && pos < *end)
2098}
2099
2100const MIN_SPLIT_RATIO: f64 = 0.3;
2103
2104fn split_at_clause_punctuation(
2108 text: &str,
2109 line_length: usize,
2110 element_spans: &[(usize, usize)],
2111 length_mode: ReflowLengthMode,
2112) -> Option<(String, String)> {
2113 let chars: Vec<char> = text.chars().collect();
2114 let min_first_len = ((line_length as f64) * MIN_SPLIT_RATIO) as usize;
2115
2116 let mut width_acc = 0;
2118 let mut search_end_char = 0;
2119 for (idx, &c) in chars.iter().enumerate() {
2120 let c_width = display_len(&c.to_string(), length_mode);
2121 if width_acc + c_width > line_length {
2122 break;
2123 }
2124 width_acc += c_width;
2125 search_end_char = idx + 1;
2126 }
2127
2128 let mut paren_depth: i32 = 0;
2135 let mut best_pos = None;
2136 for i in (0..search_end_char).rev() {
2137 let byte_start: usize = chars[..i].iter().map(|c| c.len_utf8()).sum();
2139 let byte_after: usize = byte_start + chars[i].len_utf8();
2141
2142 if !is_inside_element(byte_start, element_spans) {
2143 match chars[i] {
2144 ')' => paren_depth += 1,
2145 '(' => paren_depth = paren_depth.saturating_sub(1),
2146 _ => {}
2147 }
2148 }
2149
2150 if paren_depth == 0
2151 && is_clause_punctuation(chars[i])
2152 && clause_break_allowed_after(&chars, i)
2153 && !is_inside_element(byte_after, element_spans)
2154 {
2155 best_pos = Some(i);
2156 break;
2157 }
2158 }
2159
2160 let pos = best_pos?;
2161
2162 let first: String = chars[..=pos].iter().collect();
2164 let first_display_len = display_len(&first, length_mode);
2165 if first_display_len < min_first_len {
2166 return None;
2167 }
2168
2169 let rest: String = chars[pos + 1..].iter().collect();
2171 let rest = rest.trim_start().to_string();
2172
2173 if rest.is_empty() {
2174 return None;
2175 }
2176
2177 Some((first, rest))
2178}
2179
2180fn paren_depth_map(text: &str, element_spans: &[(usize, usize)]) -> Vec<i32> {
2187 let mut map = vec![0i32; text.len()];
2188 let mut depth = 0i32;
2189 for (byte, c) in text.char_indices() {
2190 if !is_inside_element(byte, element_spans) {
2191 match c {
2192 '(' => depth += 1,
2193 ')' => depth = depth.saturating_sub(1),
2194 _ => {}
2195 }
2196 }
2197 let end = (byte + c.len_utf8()).min(map.len());
2199 for slot in &mut map[byte..end] {
2200 *slot = depth;
2201 }
2202 }
2203 map
2204}
2205
2206fn is_standalone_parenthetical(line: &str) -> bool {
2215 let trimmed = line.trim();
2216 if !trimmed.starts_with('(') {
2217 return false;
2218 }
2219 let core = trimmed.trim_end_matches(|c: char| is_clause_punctuation(c));
2221 if !core.ends_with(')') {
2222 return false;
2223 }
2224 let inner = &core[1..core.len() - 1];
2226 if !inner.contains(' ') {
2227 return false;
2228 }
2229 let mut depth = 0i32;
2231 for c in core.chars() {
2232 match c {
2233 '(' => depth += 1,
2234 ')' => depth -= 1,
2235 _ => {}
2236 }
2237 if depth < 0 {
2238 return false;
2239 }
2240 }
2241 depth == 0
2242}
2243
2244fn split_at_break_word(
2248 text: &str,
2249 line_length: usize,
2250 element_spans: &[(usize, usize)],
2251 length_mode: ReflowLengthMode,
2252) -> Option<(String, String)> {
2253 let lower = text.to_lowercase();
2254 let min_first_len = ((line_length as f64) * MIN_SPLIT_RATIO) as usize;
2255 let mut best_split: Option<(usize, usize)> = None; let depth_map = paren_depth_map(text, element_spans);
2260
2261 for &word in BREAK_WORDS {
2262 let mut search_start = 0;
2263 while let Some(pos) = lower[search_start..].find(word) {
2264 let abs_pos = search_start + pos;
2265
2266 let preceded_by_space = abs_pos == 0 || text.as_bytes().get(abs_pos - 1) == Some(&b' ');
2268 let followed_by_space = text.as_bytes().get(abs_pos + word.len()) == Some(&b' ');
2269
2270 if preceded_by_space && followed_by_space {
2271 let first_part = text[..abs_pos].trim_end();
2273 let first_part_len = display_len(first_part, length_mode);
2274
2275 let inside_paren = depth_map.get(abs_pos).is_some_and(|&d| d > 0);
2277
2278 if first_part_len >= min_first_len
2279 && first_part_len <= line_length
2280 && !is_inside_element(abs_pos, element_spans)
2281 && !inside_paren
2282 {
2283 if best_split.is_none_or(|(prev_pos, _)| abs_pos > prev_pos) {
2285 best_split = Some((abs_pos, word.len()));
2286 }
2287 }
2288 }
2289
2290 search_start = abs_pos + word.len();
2291 }
2292 }
2293
2294 let (byte_start, _word_len) = best_split?;
2295
2296 let first = text[..byte_start].trim_end().to_string();
2297 let rest = text[byte_start..].to_string();
2298
2299 if first.is_empty() || rest.trim().is_empty() {
2300 return None;
2301 }
2302
2303 Some((first, rest))
2304}
2305
2306fn cascade_split_line(
2317 text: &str,
2318 line_length: usize,
2319 abbreviations: &Option<Vec<String>>,
2320 length_mode: ReflowLengthMode,
2321 attr_lists: bool,
2322 myst_roles: bool,
2323 defined_references: Option<&HashSet<String>>,
2324) -> Vec<String> {
2325 if line_length == 0 || display_len(text, length_mode) <= line_length {
2326 return vec![text.to_string()];
2327 }
2328
2329 let elements = parse_markdown_elements_inner(text, attr_lists, myst_roles, defined_references);
2330 let element_spans = compute_element_spans(&elements);
2331
2332 let rebased_spans = |start: usize| -> Vec<(usize, usize)> {
2336 if start == 0 {
2337 return element_spans.clone();
2338 }
2339 element_spans
2340 .iter()
2341 .filter(|&&(_, end)| end > start)
2342 .map(|&(s, e)| (s.saturating_sub(start), e.saturating_sub(start)))
2343 .collect()
2344 };
2345
2346 let mut result = Vec::new();
2347 let mut start = 0usize;
2348
2349 loop {
2350 let remaining = &text[start..];
2351 if display_len(remaining, length_mode) <= line_length {
2352 result.push(remaining.to_string());
2353 return result;
2354 }
2355
2356 let spans = rebased_spans(start);
2357
2358 let split = split_at_parenthetical(remaining, line_length, &spans, length_mode)
2362 .or_else(|| split_at_clause_punctuation(remaining, line_length, &spans, length_mode))
2363 .or_else(|| split_at_break_word(remaining, line_length, &spans, length_mode));
2364
2365 if let Some((first, rest)) = split {
2366 let consumed = remaining.len().saturating_sub(rest.len());
2367 if consumed == 0 {
2370 break;
2371 }
2372 result.push(first);
2373 start += consumed;
2374 continue;
2375 }
2376
2377 break;
2379 }
2380
2381 let options = ReflowOptions {
2383 line_length,
2384 break_on_sentences: false,
2385 preserve_breaks: false,
2386 sentence_per_line: false,
2387 semantic_line_breaks: false,
2388 abbreviations: abbreviations.clone(),
2389 length_mode,
2390 attr_lists,
2391 myst_roles,
2392 require_sentence_capital: true,
2393 max_list_continuation_indent: None,
2394 defined_references: None,
2397 };
2398 let remaining = &text[start..];
2399 let tail_elements = if start == 0 {
2400 elements
2401 } else {
2402 parse_markdown_elements_inner(remaining, attr_lists, myst_roles, defined_references)
2403 };
2404 result.extend(reflow_elements(&tail_elements, &options));
2405 result
2406}
2407
2408fn reflow_elements_semantic(elements: &[Element], options: &ReflowOptions) -> Vec<String> {
2412 let sentence_lines =
2414 reflow_elements_sentence_per_line(elements, &options.abbreviations, options.require_sentence_capital);
2415
2416 if options.line_length == 0 {
2419 return sentence_lines;
2420 }
2421
2422 let length_mode = options.length_mode;
2423 let mut result = Vec::new();
2424 for line in sentence_lines {
2425 if display_len(&line, length_mode) <= options.line_length {
2426 result.push(line);
2427 } else {
2428 result.extend(cascade_split_line(
2429 &line,
2430 options.line_length,
2431 &options.abbreviations,
2432 length_mode,
2433 options.attr_lists,
2434 options.myst_roles,
2435 options.defined_references.as_ref(),
2436 ));
2437 }
2438 }
2439
2440 let min_line_len = ((options.line_length as f64) * MIN_SPLIT_RATIO) as usize;
2443 let mut merged: Vec<String> = Vec::with_capacity(result.len());
2444 for line in result {
2445 if !merged.is_empty() && display_len(&line, length_mode) < min_line_len && !line.trim().is_empty() {
2446 if is_standalone_parenthetical(&line) {
2449 merged.push(line);
2450 continue;
2451 }
2452
2453 let prev_ends_at_sentence = {
2455 let trimmed = merged.last().unwrap().trim_end();
2456 trimmed
2457 .chars()
2458 .rev()
2459 .find(|c| !matches!(c, '"' | '\'' | '\u{201D}' | '\u{2019}' | ')' | ']'))
2460 .is_some_and(|c| matches!(c, '.' | '!' | '?'))
2461 };
2462
2463 if !prev_ends_at_sentence {
2464 let prev = merged.last_mut().unwrap();
2465 let combined = format!("{prev} {line}");
2466 if display_len(&combined, length_mode) <= options.line_length {
2468 *prev = combined;
2469 continue;
2470 }
2471 }
2472 }
2473 merged.push(line);
2474 }
2475 merged
2476}
2477
2478fn rfind_safe_space(line: &str, element_spans: &[(usize, usize)]) -> Option<usize> {
2488 line.char_indices().rev().map(|(pos, _)| pos).find(|&pos| {
2489 line.as_bytes()[pos] == b' '
2490 && !element_spans.iter().any(|(s, e)| pos > *s && pos < *e)
2491 && !starts_block_construct(&line[pos + 1..])
2492 })
2493}
2494
2495fn reflow_elements(elements: &[Element], options: &ReflowOptions) -> Vec<String> {
2497 let mut lines = Vec::new();
2498 let mut current_line = String::new();
2499 let mut current_length = 0;
2500 let mut current_line_element_spans: Vec<(usize, usize)> = Vec::new();
2502 let length_mode = options.length_mode;
2503
2504 for (idx, element) in elements.iter().enumerate() {
2505 let element_str = format!("{element}");
2508 let element_len = display_len(&element_str, length_mode);
2509
2510 let is_adjacent_to_prev = if idx > 0 {
2516 match (&elements[idx - 1], element) {
2517 (Element::Text(t), _) => !t.is_empty() && !t.ends_with(char::is_whitespace),
2518 (_, Element::Text(t)) => !t.is_empty() && !t.starts_with(char::is_whitespace),
2519 _ => true,
2520 }
2521 } else {
2522 false
2523 };
2524
2525 if let Element::Text(text) = element {
2527 let has_leading_space = text.starts_with(char::is_whitespace);
2529 let words: Vec<&str> = text.split_whitespace().collect();
2531
2532 for (i, word) in words.iter().enumerate() {
2533 let word_len = display_len(word, length_mode);
2534 let is_trailing_punct = word
2536 .chars()
2537 .all(|c| matches!(c, ',' | '.' | ':' | ';' | '!' | '?' | ')' | ']' | '}'));
2538
2539 let is_first_adjacent = i == 0 && is_adjacent_to_prev;
2542
2543 if is_first_adjacent {
2544 if current_length + word_len > options.line_length && current_length > 0 {
2546 if let Some(last_space) = rfind_safe_space(¤t_line, ¤t_line_element_spans) {
2550 let before = current_line[..last_space].trim_end().to_string();
2551 let after = current_line[last_space + 1..].to_string();
2552 lines.push(before);
2553 current_line = format!("{after}{word}");
2554 current_length = display_len(¤t_line, length_mode);
2555 current_line_element_spans.clear();
2556 } else {
2557 current_line.push_str(word);
2558 current_length += word_len;
2559 }
2560 } else {
2561 current_line.push_str(word);
2562 current_length += word_len;
2563 }
2564 } else if current_length > 0
2565 && current_length + 1 + word_len > options.line_length
2566 && !is_trailing_punct
2567 {
2568 if !starts_block_construct(word) {
2569 lines.push(current_line.trim().to_string());
2571 current_line = word.to_string();
2572 current_length = word_len;
2573 current_line_element_spans.clear();
2574 } else if let Some(last_space) = rfind_safe_space(¤t_line, ¤t_line_element_spans) {
2575 let before = current_line[..last_space].trim_end().to_string();
2580 let after = current_line[last_space + 1..].to_string();
2581 lines.push(before);
2582 current_line = format!("{after} {word}");
2583 current_length = display_len(¤t_line, length_mode);
2584 current_line_element_spans.clear();
2585 } else {
2586 if i > 0 || has_leading_space {
2589 current_line.push(' ');
2590 current_length += 1;
2591 }
2592 current_line.push_str(word);
2593 current_length += word_len;
2594 }
2595 } else {
2596 let add_space = current_length > 0 && if i == 0 { has_leading_space } else { !is_trailing_punct };
2606 if add_space {
2607 current_line.push(' ');
2608 current_length += 1;
2609 }
2610 current_line.push_str(word);
2611 current_length += word_len;
2612 }
2613 }
2614 } else if matches!(
2615 element,
2616 Element::Italic { .. } | Element::Bold { .. } | Element::Strikethrough { .. }
2617 ) && element_len > options.line_length
2618 {
2619 let (content, marker): (&str, &str) = match element {
2623 Element::Italic { content, underscore } => (content.as_str(), if *underscore { "_" } else { "*" }),
2624 Element::Bold { content, underscore } => (content.as_str(), if *underscore { "__" } else { "**" }),
2625 Element::Strikethrough { content, double } => (content.as_str(), if *double { "~~" } else { "~" }),
2626 _ => unreachable!(),
2627 };
2628
2629 let words: Vec<&str> = content.split_whitespace().collect();
2630 let n = words.len();
2631
2632 if n == 0 {
2633 let full = format!("{marker}{marker}");
2635 let full_len = display_len(&full, length_mode);
2636 if !is_adjacent_to_prev && current_length > 0 {
2637 current_line.push(' ');
2638 current_length += 1;
2639 }
2640 current_line.push_str(&full);
2641 current_length += full_len;
2642 } else {
2643 for (i, word) in words.iter().enumerate() {
2644 let is_first = i == 0;
2645 let is_last = i == n - 1;
2646 let word_str: String = match (is_first, is_last) {
2647 (true, true) => format!("{marker}{word}{marker}"),
2648 (true, false) => format!("{marker}{word}"),
2649 (false, true) => format!("{word}{marker}"),
2650 (false, false) => word.to_string(),
2651 };
2652 let word_len = display_len(&word_str, length_mode);
2653
2654 let needs_space = if is_first {
2655 !is_adjacent_to_prev && current_length > 0
2656 } else {
2657 current_length > 0
2658 };
2659
2660 if needs_space
2661 && current_length + 1 + word_len > options.line_length
2662 && !starts_block_construct(&word_str)
2663 {
2664 lines.push(current_line.trim_end().to_string());
2665 current_line = word_str;
2666 current_length = word_len;
2667 current_line_element_spans.clear();
2668 } else {
2669 if needs_space {
2670 current_line.push(' ');
2671 current_length += 1;
2672 }
2673 current_line.push_str(&word_str);
2674 current_length += word_len;
2675 }
2676 }
2677 }
2678 } else {
2679 if is_adjacent_to_prev {
2683 if current_length + element_len > options.line_length {
2685 if let Some(last_space) = rfind_safe_space(¤t_line, ¤t_line_element_spans) {
2689 let before = current_line[..last_space].trim_end().to_string();
2690 let after = current_line[last_space + 1..].to_string();
2691 lines.push(before);
2692 current_line = format!("{after}{element_str}");
2693 current_length = display_len(¤t_line, length_mode);
2694 current_line_element_spans.clear();
2695 let start = after.len();
2697 current_line_element_spans.push((start, start + element_str.len()));
2698 } else {
2699 let start = current_line.len();
2701 current_line.push_str(&element_str);
2702 current_length += element_len;
2703 current_line_element_spans.push((start, current_line.len()));
2704 }
2705 } else {
2706 let start = current_line.len();
2707 current_line.push_str(&element_str);
2708 current_length += element_len;
2709 current_line_element_spans.push((start, current_line.len()));
2710 }
2711 } else if current_length > 0 && current_length + 1 + element_len > options.line_length {
2712 if !starts_block_construct(&element_str) {
2713 lines.push(current_line.trim().to_string());
2715 current_line.clone_from(&element_str);
2716 current_length = element_len;
2717 current_line_element_spans.clear();
2718 current_line_element_spans.push((0, element_str.len()));
2719 } else if let Some(last_space) = rfind_safe_space(¤t_line, ¤t_line_element_spans) {
2720 let before = current_line[..last_space].trim_end().to_string();
2724 let after = current_line[last_space + 1..].to_string();
2725 lines.push(before);
2726 current_line = format!("{after} {element_str}");
2727 current_length = display_len(¤t_line, length_mode);
2728 current_line_element_spans.clear();
2729 let start = after.len() + 1;
2730 current_line_element_spans.push((start, start + element_str.len()));
2731 } else {
2732 let ends_with_opener =
2735 current_line.ends_with('(') || current_line.ends_with('[') || current_line.ends_with('{');
2736 if !ends_with_opener {
2737 current_line.push(' ');
2738 current_length += 1;
2739 }
2740 let start = current_line.len();
2741 current_line.push_str(&element_str);
2742 current_length += element_len;
2743 current_line_element_spans.push((start, current_line.len()));
2744 }
2745 } else {
2746 let ends_with_opener =
2748 current_line.ends_with('(') || current_line.ends_with('[') || current_line.ends_with('{');
2749 if current_length > 0 && !ends_with_opener {
2750 current_line.push(' ');
2751 current_length += 1;
2752 }
2753 let start = current_line.len();
2754 current_line.push_str(&element_str);
2755 current_length += element_len;
2756 current_line_element_spans.push((start, current_line.len()));
2757 }
2758 }
2759 }
2760
2761 if !current_line.is_empty() {
2763 lines.push(current_line.trim_end().to_string());
2764 }
2765
2766 lines
2767}
2768
2769pub fn reflow_markdown(content: &str, options: &ReflowOptions) -> String {
2771 let lines: Vec<&str> = content.lines().collect();
2772 let mut result = Vec::new();
2773 let mut i = 0;
2774
2775 while i < lines.len() {
2776 let line = lines[i];
2777 let trimmed = line.trim();
2778
2779 if trimmed.is_empty() {
2781 result.push(String::new());
2782 i += 1;
2783 continue;
2784 }
2785
2786 if trimmed.starts_with('#') {
2788 result.push(line.to_string());
2789 i += 1;
2790 continue;
2791 }
2792
2793 if trimmed.starts_with(":::") {
2795 result.push(line.to_string());
2796 i += 1;
2797 continue;
2798 }
2799
2800 if trimmed.starts_with("```") || trimmed.starts_with("~~~") {
2802 result.push(line.to_string());
2803 i += 1;
2804 while i < lines.len() {
2806 result.push(lines[i].to_string());
2807 if lines[i].trim().starts_with("```") || lines[i].trim().starts_with("~~~") {
2808 i += 1;
2809 break;
2810 }
2811 i += 1;
2812 }
2813 continue;
2814 }
2815
2816 if calculate_indentation_width_default(line) >= 4 {
2818 result.push(line.to_string());
2820 i += 1;
2821 while i < lines.len() {
2822 let next_line = lines[i];
2823 if calculate_indentation_width_default(next_line) >= 4 || next_line.trim().is_empty() {
2825 result.push(next_line.to_string());
2826 i += 1;
2827 } else {
2828 break;
2829 }
2830 }
2831 continue;
2832 }
2833
2834 if trimmed.starts_with('>') {
2836 let gt_pos = line.find('>').expect("'>' must exist since trimmed.starts_with('>')");
2839 let quote_prefix = line[0..=gt_pos].to_string();
2840 let quote_content = &line[quote_prefix.len()..].trim_start();
2841
2842 let reflowed = reflow_line(quote_content, options);
2843 for reflowed_line in &reflowed {
2844 result.push(format!("{quote_prefix} {reflowed_line}"));
2845 }
2846 i += 1;
2847 continue;
2848 }
2849
2850 if is_horizontal_rule(trimmed) {
2852 result.push(line.to_string());
2853 i += 1;
2854 continue;
2855 }
2856
2857 if is_unordered_list_marker(trimmed) || is_numbered_list_item(trimmed) {
2859 let indent = line.len() - line.trim_start().len();
2861 let indent_str = " ".repeat(indent);
2862
2863 let mut marker_end = indent;
2866 let mut content_start = indent;
2867
2868 if trimmed.chars().next().is_some_and(char::is_numeric) {
2869 if let Some(period_pos) = line[indent..].find('.') {
2871 marker_end = indent + period_pos + 1; content_start = marker_end;
2873 while content_start < line.len() && line.as_bytes().get(content_start) == Some(&b' ') {
2877 content_start += 1;
2878 }
2879 }
2880 } else {
2881 marker_end = indent + 1; content_start = marker_end;
2884 while content_start < line.len() && line.as_bytes().get(content_start) == Some(&b' ') {
2888 content_start += 1;
2889 }
2890 }
2891
2892 let min_continuation_indent = content_start;
2894
2895 let rest = &line[content_start..];
2898 if rest.starts_with("[ ] ") || rest.starts_with("[x] ") || rest.starts_with("[X] ") {
2899 marker_end = content_start + 3; content_start += 4; }
2902
2903 let marker = &line[indent..marker_end];
2904
2905 let mut list_content = vec![trim_preserving_hard_break(&line[content_start..])];
2908 i += 1;
2909
2910 while i < lines.len() {
2914 let next_line = lines[i];
2915 let next_trimmed = next_line.trim();
2916
2917 if is_block_boundary(next_trimmed) {
2919 break;
2920 }
2921
2922 let next_indent = next_line.len() - next_line.trim_start().len();
2924 if next_indent >= min_continuation_indent {
2925 let trimmed_start = next_line.trim_start();
2928 list_content.push(trim_preserving_hard_break(trimmed_start));
2929 i += 1;
2930 } else {
2931 break;
2933 }
2934 }
2935
2936 let combined_content = if options.preserve_breaks {
2939 list_content[0].clone()
2940 } else {
2941 let has_hard_breaks = list_content.iter().any(|line| has_hard_break(line));
2943 if has_hard_breaks {
2944 list_content.join("\n")
2946 } else {
2947 list_content.join(" ")
2949 }
2950 };
2951
2952 let trimmed_marker = marker;
2954 let continuation_spaces = if let Some(max_indent) = options.max_list_continuation_indent {
2955 indent + (content_start - indent).min(max_indent)
2958 } else {
2959 content_start
2960 };
2961
2962 let prefix_length = indent + trimmed_marker.len() + 1;
2964
2965 let adjusted_options = ReflowOptions {
2967 line_length: options.line_length.saturating_sub(prefix_length),
2968 ..options.clone()
2969 };
2970
2971 let reflowed = reflow_line(&combined_content, &adjusted_options);
2972 for (j, reflowed_line) in reflowed.iter().enumerate() {
2973 if j == 0 {
2974 result.push(format!("{indent_str}{trimmed_marker} {reflowed_line}"));
2975 } else {
2976 let continuation_indent = " ".repeat(continuation_spaces);
2978 result.push(format!("{continuation_indent}{reflowed_line}"));
2979 }
2980 }
2981 continue;
2982 }
2983
2984 if crate::utils::table_utils::TableUtils::is_potential_table_row(line) {
2986 result.push(line.to_string());
2987 i += 1;
2988 continue;
2989 }
2990
2991 if trimmed.starts_with('[') && line.contains("]:") {
2993 result.push(line.to_string());
2994 i += 1;
2995 continue;
2996 }
2997
2998 if is_definition_list_item(trimmed) {
3000 result.push(line.to_string());
3001 i += 1;
3002 continue;
3003 }
3004
3005 let mut is_single_line_paragraph = true;
3007 if i + 1 < lines.len() {
3008 let next_trimmed = lines[i + 1].trim();
3009 if !is_block_boundary(next_trimmed) {
3011 is_single_line_paragraph = false;
3012 }
3013 }
3014
3015 if is_single_line_paragraph && display_len(line, options.length_mode) <= options.line_length {
3017 result.push(line.to_string());
3018 i += 1;
3019 continue;
3020 }
3021
3022 let mut paragraph_parts = Vec::new();
3024 let mut current_part = vec![line];
3025 i += 1;
3026
3027 if options.preserve_breaks {
3029 let hard_break_type = if line.strip_suffix('\r').unwrap_or(line).ends_with('\\') {
3031 Some("\\")
3032 } else if line.ends_with(" ") {
3033 Some(" ")
3034 } else {
3035 None
3036 };
3037 let reflowed = reflow_line(line, options);
3038
3039 if let Some(break_marker) = hard_break_type {
3041 if !reflowed.is_empty() {
3042 let mut reflowed_with_break = reflowed;
3043 let last_idx = reflowed_with_break.len() - 1;
3044 if !has_hard_break(&reflowed_with_break[last_idx]) {
3045 reflowed_with_break[last_idx].push_str(break_marker);
3046 }
3047 result.extend(reflowed_with_break);
3048 }
3049 } else {
3050 result.extend(reflowed);
3051 }
3052 } else {
3053 while i < lines.len() {
3055 let prev_line = if !current_part.is_empty() {
3056 current_part.last().unwrap()
3057 } else {
3058 ""
3059 };
3060 let next_line = lines[i];
3061 let next_trimmed = next_line.trim();
3062
3063 if is_block_boundary(next_trimmed) {
3065 break;
3066 }
3067
3068 let prev_trimmed = prev_line.trim();
3071 let abbreviations = get_abbreviations(&options.abbreviations);
3072 let ends_with_sentence = (prev_trimmed.ends_with('.')
3073 || prev_trimmed.ends_with('!')
3074 || prev_trimmed.ends_with('?')
3075 || prev_trimmed.ends_with(".*")
3076 || prev_trimmed.ends_with("!*")
3077 || prev_trimmed.ends_with("?*")
3078 || prev_trimmed.ends_with("._")
3079 || prev_trimmed.ends_with("!_")
3080 || prev_trimmed.ends_with("?_")
3081 || prev_trimmed.ends_with(".\"")
3083 || prev_trimmed.ends_with("!\"")
3084 || prev_trimmed.ends_with("?\"")
3085 || prev_trimmed.ends_with(".'")
3086 || prev_trimmed.ends_with("!'")
3087 || prev_trimmed.ends_with("?'")
3088 || prev_trimmed.ends_with(".\u{201D}")
3089 || prev_trimmed.ends_with("!\u{201D}")
3090 || prev_trimmed.ends_with("?\u{201D}")
3091 || prev_trimmed.ends_with(".\u{2019}")
3092 || prev_trimmed.ends_with("!\u{2019}")
3093 || prev_trimmed.ends_with("?\u{2019}"))
3094 && !text_ends_with_abbreviation(
3095 prev_trimmed.trim_end_matches(['*', '_', '"', '\'', '\u{201D}', '\u{2019}']),
3096 &abbreviations,
3097 );
3098
3099 if has_hard_break(prev_line) || (options.sentence_per_line && ends_with_sentence) {
3100 paragraph_parts.push(current_part.join(" "));
3102 current_part = vec![next_line];
3103 } else {
3104 current_part.push(next_line);
3105 }
3106 i += 1;
3107 }
3108
3109 if !current_part.is_empty() {
3111 if current_part.len() == 1 {
3112 paragraph_parts.push(current_part[0].to_string());
3114 } else {
3115 paragraph_parts.push(current_part.join(" "));
3116 }
3117 }
3118
3119 for (j, part) in paragraph_parts.iter().enumerate() {
3121 let reflowed = reflow_line(part, options);
3122 result.extend(reflowed);
3123
3124 if j < paragraph_parts.len() - 1 && !result.is_empty() && !options.sentence_per_line {
3128 let last_idx = result.len() - 1;
3129 if !has_hard_break(&result[last_idx]) {
3130 result[last_idx].push_str(" ");
3131 }
3132 }
3133 }
3134 }
3135 }
3136
3137 let result_text = result.join("\n");
3139 if content.ends_with('\n') && !result_text.ends_with('\n') {
3140 format!("{result_text}\n")
3141 } else {
3142 result_text
3143 }
3144}
3145
3146#[derive(Debug, Clone)]
3148pub struct ParagraphReflow {
3149 pub start_byte: usize,
3151 pub end_byte: usize,
3153 pub reflowed_text: String,
3155}
3156
3157#[derive(Debug, Clone)]
3163pub struct BlockquoteLineData {
3164 pub(crate) content: String,
3166 pub(crate) is_explicit: bool,
3168 pub(crate) prefix: Option<String>,
3170}
3171
3172impl BlockquoteLineData {
3173 pub fn explicit(content: String, prefix: String) -> Self {
3175 Self {
3176 content,
3177 is_explicit: true,
3178 prefix: Some(prefix),
3179 }
3180 }
3181
3182 pub fn lazy(content: String) -> Self {
3184 Self {
3185 content,
3186 is_explicit: false,
3187 prefix: None,
3188 }
3189 }
3190}
3191
3192#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3194pub enum BlockquoteContinuationStyle {
3195 Explicit,
3196 Lazy,
3197}
3198
3199pub fn blockquote_continuation_style(lines: &[BlockquoteLineData]) -> BlockquoteContinuationStyle {
3207 let mut explicit_count = 0usize;
3208 let mut lazy_count = 0usize;
3209
3210 for line in lines.iter().skip(1) {
3211 if line.is_explicit {
3212 explicit_count += 1;
3213 } else {
3214 lazy_count += 1;
3215 }
3216 }
3217
3218 if explicit_count > 0 && lazy_count == 0 {
3219 BlockquoteContinuationStyle::Explicit
3220 } else if lazy_count > 0 && explicit_count == 0 {
3221 BlockquoteContinuationStyle::Lazy
3222 } else if explicit_count >= lazy_count {
3223 BlockquoteContinuationStyle::Explicit
3224 } else {
3225 BlockquoteContinuationStyle::Lazy
3226 }
3227}
3228
3229pub fn dominant_blockquote_prefix(lines: &[BlockquoteLineData], fallback: &str) -> String {
3234 let mut counts: std::collections::HashMap<String, (usize, usize)> = std::collections::HashMap::new();
3235
3236 for (idx, line) in lines.iter().enumerate() {
3237 let Some(prefix) = line.prefix.as_ref() else {
3238 continue;
3239 };
3240 counts
3241 .entry(prefix.clone())
3242 .and_modify(|entry| entry.0 += 1)
3243 .or_insert((1, idx));
3244 }
3245
3246 counts
3247 .into_iter()
3248 .max_by(|(_, (count_a, first_idx_a)), (_, (count_b, first_idx_b))| {
3249 count_a.cmp(count_b).then_with(|| first_idx_b.cmp(first_idx_a))
3250 })
3251 .map_or_else(|| fallback.to_string(), |(prefix, _)| prefix)
3252}
3253
3254pub(crate) fn should_force_explicit_blockquote_line(content_line: &str) -> bool {
3259 let trimmed = content_line.trim_start();
3260 trimmed.starts_with('>')
3261 || trimmed.starts_with('#')
3262 || trimmed.starts_with("```")
3263 || trimmed.starts_with("~~~")
3264 || is_unordered_list_marker(trimmed)
3265 || is_numbered_list_item(trimmed)
3266 || is_horizontal_rule(trimmed)
3267 || is_definition_list_item(trimmed)
3268 || (trimmed.starts_with('[') && trimmed.contains("]:"))
3269 || trimmed.starts_with(":::")
3270 || (trimmed.starts_with('<')
3271 && !trimmed.starts_with("<http")
3272 && !trimmed.starts_with("<https")
3273 && !trimmed.starts_with("<mailto:"))
3274}
3275
3276pub fn reflow_blockquote_content(
3285 lines: &[BlockquoteLineData],
3286 explicit_prefix: &str,
3287 continuation_style: BlockquoteContinuationStyle,
3288 options: &ReflowOptions,
3289) -> Vec<String> {
3290 let content_strs: Vec<&str> = lines.iter().map(|l| l.content.as_str()).collect();
3291 let segments = split_into_segments_strs(&content_strs);
3292 let mut reflowed_content_lines: Vec<String> = Vec::new();
3293
3294 for segment in segments {
3295 let hard_break_type = segment.last().and_then(|&line| {
3296 let line = line.strip_suffix('\r').unwrap_or(line);
3297 if line.ends_with('\\') {
3298 Some("\\")
3299 } else if line.ends_with(" ") {
3300 Some(" ")
3301 } else {
3302 None
3303 }
3304 });
3305
3306 let pieces: Vec<&str> = segment
3307 .iter()
3308 .map(|&line| {
3309 if let Some(l) = line.strip_suffix('\\') {
3310 l.trim_end()
3311 } else if let Some(l) = line.strip_suffix(" ") {
3312 l.trim_end()
3313 } else {
3314 line.trim_end()
3315 }
3316 })
3317 .collect();
3318
3319 let segment_text = pieces.join(" ");
3320 let segment_text = segment_text.trim();
3321 if segment_text.is_empty() {
3322 continue;
3323 }
3324
3325 let mut reflowed = reflow_line(segment_text, options);
3326 if let Some(break_marker) = hard_break_type
3327 && !reflowed.is_empty()
3328 {
3329 let last_idx = reflowed.len() - 1;
3330 if !has_hard_break(&reflowed[last_idx]) {
3331 reflowed[last_idx].push_str(break_marker);
3332 }
3333 }
3334 reflowed_content_lines.extend(reflowed);
3335 }
3336
3337 let mut styled_lines: Vec<String> = Vec::new();
3338 for (idx, line) in reflowed_content_lines.iter().enumerate() {
3339 let force_explicit = idx == 0
3340 || continuation_style == BlockquoteContinuationStyle::Explicit
3341 || should_force_explicit_blockquote_line(line);
3342 if force_explicit {
3343 styled_lines.push(format!("{explicit_prefix}{line}"));
3344 } else {
3345 styled_lines.push(line.clone());
3346 }
3347 }
3348
3349 styled_lines
3350}
3351
3352fn is_blockquote_content_boundary(content: &str) -> bool {
3353 let trimmed = content.trim();
3354 trimmed.is_empty()
3355 || is_block_boundary(trimmed)
3356 || crate::utils::table_utils::TableUtils::is_potential_table_row(content)
3357 || trimmed.starts_with(":::")
3358 || crate::utils::is_template_directive_only(content)
3359 || is_standalone_attr_list(content)
3360 || is_snippet_block_delimiter(content)
3361}
3362
3363fn split_into_segments_strs<'a>(lines: &[&'a str]) -> Vec<Vec<&'a str>> {
3364 let mut segments = Vec::new();
3365 let mut current = Vec::new();
3366
3367 for &line in lines {
3368 current.push(line);
3369 if has_hard_break(line) {
3370 segments.push(current);
3371 current = Vec::new();
3372 }
3373 }
3374
3375 if !current.is_empty() {
3376 segments.push(current);
3377 }
3378
3379 segments
3380}
3381
3382fn reflow_blockquote_paragraph_at_line(
3383 content: &str,
3384 lines: &[&str],
3385 target_idx: usize,
3386 options: &ReflowOptions,
3387) -> Option<ParagraphReflow> {
3388 let mut anchor_idx = target_idx;
3389 let mut target_level = if let Some(parsed) = crate::utils::blockquote::parse_blockquote_prefix(lines[target_idx]) {
3390 parsed.nesting_level
3391 } else {
3392 let mut found = None;
3393 let mut idx = target_idx;
3394 loop {
3395 if lines[idx].trim().is_empty() {
3396 break;
3397 }
3398 if let Some(parsed) = crate::utils::blockquote::parse_blockquote_prefix(lines[idx]) {
3399 found = Some((idx, parsed.nesting_level));
3400 break;
3401 }
3402 if idx == 0 {
3403 break;
3404 }
3405 idx -= 1;
3406 }
3407 let (idx, level) = found?;
3408 anchor_idx = idx;
3409 level
3410 };
3411
3412 let mut para_start = anchor_idx;
3414 while para_start > 0 {
3415 let prev_idx = para_start - 1;
3416 let prev_line = lines[prev_idx];
3417
3418 if prev_line.trim().is_empty() {
3419 break;
3420 }
3421
3422 if let Some(parsed) = crate::utils::blockquote::parse_blockquote_prefix(prev_line) {
3423 if parsed.nesting_level != target_level || is_blockquote_content_boundary(parsed.content) {
3424 break;
3425 }
3426 para_start = prev_idx;
3427 continue;
3428 }
3429
3430 let prev_lazy = prev_line.trim_start();
3431 if is_blockquote_content_boundary(prev_lazy) {
3432 break;
3433 }
3434 para_start = prev_idx;
3435 }
3436
3437 while para_start < lines.len() {
3439 let Some(parsed) = crate::utils::blockquote::parse_blockquote_prefix(lines[para_start]) else {
3440 para_start += 1;
3441 continue;
3442 };
3443 target_level = parsed.nesting_level;
3444 break;
3445 }
3446
3447 if para_start >= lines.len() || para_start > target_idx {
3448 return None;
3449 }
3450
3451 let mut collected: Vec<(usize, BlockquoteLineData)> = Vec::new();
3454 let mut idx = para_start;
3455 while idx < lines.len() {
3456 if !collected.is_empty() && has_hard_break(&collected[collected.len() - 1].1.content) {
3457 break;
3458 }
3459
3460 let line = lines[idx];
3461 if line.trim().is_empty() {
3462 break;
3463 }
3464
3465 if let Some(parsed) = crate::utils::blockquote::parse_blockquote_prefix(line) {
3466 if parsed.nesting_level != target_level || is_blockquote_content_boundary(parsed.content) {
3467 break;
3468 }
3469 collected.push((
3470 idx,
3471 BlockquoteLineData::explicit(trim_preserving_hard_break(parsed.content), parsed.prefix.to_string()),
3472 ));
3473 idx += 1;
3474 continue;
3475 }
3476
3477 let lazy_content = line.trim_start();
3478 if is_blockquote_content_boundary(lazy_content) {
3479 break;
3480 }
3481
3482 collected.push((idx, BlockquoteLineData::lazy(trim_preserving_hard_break(lazy_content))));
3483 idx += 1;
3484 }
3485
3486 if collected.is_empty() {
3487 return None;
3488 }
3489
3490 let para_end = collected[collected.len() - 1].0;
3491 if target_idx < para_start || target_idx > para_end {
3492 return None;
3493 }
3494
3495 let line_data: Vec<BlockquoteLineData> = collected.iter().map(|(_, d)| d.clone()).collect();
3496
3497 let fallback_prefix = line_data
3498 .iter()
3499 .find_map(|d| d.prefix.clone())
3500 .unwrap_or_else(|| "> ".to_string());
3501 let explicit_prefix = dominant_blockquote_prefix(&line_data, &fallback_prefix);
3502 let continuation_style = blockquote_continuation_style(&line_data);
3503
3504 let adjusted_line_length = options
3505 .line_length
3506 .saturating_sub(display_len(&explicit_prefix, options.length_mode))
3507 .max(1);
3508
3509 let adjusted_options = ReflowOptions {
3510 line_length: adjusted_line_length,
3511 ..options.clone()
3512 };
3513
3514 let styled_lines = reflow_blockquote_content(&line_data, &explicit_prefix, continuation_style, &adjusted_options);
3515
3516 if styled_lines.is_empty() {
3517 return None;
3518 }
3519
3520 let mut start_byte = 0;
3522 for line in lines.iter().take(para_start) {
3523 start_byte += line.len() + 1;
3524 }
3525
3526 let mut end_byte = start_byte;
3527 for line in lines.iter().take(para_end + 1).skip(para_start) {
3528 end_byte += line.len() + 1;
3529 }
3530
3531 let includes_trailing_newline = para_end != lines.len() - 1 || content.ends_with('\n');
3532 if !includes_trailing_newline {
3533 end_byte -= 1;
3534 }
3535
3536 let reflowed_joined = styled_lines.join("\n");
3537 let reflowed_text = if includes_trailing_newline {
3538 if reflowed_joined.ends_with('\n') {
3539 reflowed_joined
3540 } else {
3541 format!("{reflowed_joined}\n")
3542 }
3543 } else if reflowed_joined.ends_with('\n') {
3544 reflowed_joined.trim_end_matches('\n').to_string()
3545 } else {
3546 reflowed_joined
3547 };
3548
3549 Some(ParagraphReflow {
3550 start_byte,
3551 end_byte,
3552 reflowed_text,
3553 })
3554}
3555
3556pub fn reflow_paragraph_at_line(content: &str, line_number: usize, line_length: usize) -> Option<ParagraphReflow> {
3574 reflow_paragraph_at_line_with_mode(content, line_number, line_length, ReflowLengthMode::default())
3575}
3576
3577pub fn reflow_paragraph_at_line_with_mode(
3579 content: &str,
3580 line_number: usize,
3581 line_length: usize,
3582 length_mode: ReflowLengthMode,
3583) -> Option<ParagraphReflow> {
3584 let options = ReflowOptions {
3585 line_length,
3586 length_mode,
3587 ..Default::default()
3588 };
3589 reflow_paragraph_at_line_with_options(content, line_number, &options)
3590}
3591
3592pub fn reflow_paragraph_at_line_with_options(
3603 content: &str,
3604 line_number: usize,
3605 options: &ReflowOptions,
3606) -> Option<ParagraphReflow> {
3607 if line_number == 0 {
3608 return None;
3609 }
3610
3611 let lines: Vec<&str> = content.lines().collect();
3612
3613 if line_number > lines.len() {
3615 return None;
3616 }
3617
3618 let target_idx = line_number - 1; let target_line = lines[target_idx];
3620 let trimmed = target_line.trim();
3621
3622 if let Some(blockquote_reflow) = reflow_blockquote_paragraph_at_line(content, &lines, target_idx, options) {
3625 return Some(blockquote_reflow);
3626 }
3627
3628 if is_paragraph_boundary(trimmed, target_line) {
3630 return None;
3631 }
3632
3633 let mut para_start = target_idx;
3635 while para_start > 0 {
3636 let prev_idx = para_start - 1;
3637 let prev_line = lines[prev_idx];
3638 let prev_trimmed = prev_line.trim();
3639
3640 if is_paragraph_boundary(prev_trimmed, prev_line) {
3642 break;
3643 }
3644
3645 para_start = prev_idx;
3646 }
3647
3648 let mut para_end = target_idx;
3650 while para_end + 1 < lines.len() {
3651 let next_idx = para_end + 1;
3652 let next_line = lines[next_idx];
3653 let next_trimmed = next_line.trim();
3654
3655 if is_paragraph_boundary(next_trimmed, next_line) {
3657 break;
3658 }
3659
3660 para_end = next_idx;
3661 }
3662
3663 let paragraph_lines = &lines[para_start..=para_end];
3665
3666 let mut start_byte = 0;
3668 for line in lines.iter().take(para_start) {
3669 start_byte += line.len() + 1; }
3671
3672 let mut end_byte = start_byte;
3673 for line in paragraph_lines {
3674 end_byte += line.len() + 1; }
3676
3677 let includes_trailing_newline = para_end != lines.len() - 1 || content.ends_with('\n');
3680
3681 if !includes_trailing_newline {
3683 end_byte -= 1;
3684 }
3685
3686 let paragraph_text = paragraph_lines.join("\n");
3688
3689 let reflowed = reflow_markdown(¶graph_text, options);
3691
3692 let reflowed_text = if includes_trailing_newline {
3696 if reflowed.ends_with('\n') {
3698 reflowed
3699 } else {
3700 format!("{reflowed}\n")
3701 }
3702 } else {
3703 if reflowed.ends_with('\n') {
3705 reflowed.trim_end_matches('\n').to_string()
3706 } else {
3707 reflowed
3708 }
3709 };
3710
3711 Some(ParagraphReflow {
3712 start_byte,
3713 end_byte,
3714 reflowed_text,
3715 })
3716}
3717
3718#[cfg(test)]
3719mod tests {
3720 use super::*;
3721
3722 #[test]
3723 fn cascade_split_line_handles_a_very_long_line_without_overflowing() {
3724 let words: Vec<String> = (0..4000).map(|i| format!("word{i}")).collect();
3730 let line = words.join(" ");
3731
3732 let out = cascade_split_line(&line, 80, &None, ReflowLengthMode::Chars, false, false, None);
3733
3734 assert!(out.len() > 1, "a very long line should split into many lines");
3735 for segment in &out {
3736 assert!(
3737 display_len(segment, ReflowLengthMode::Chars) <= 80 || !segment.contains(' '),
3738 "each wrapped line should fit the width (or be a single unbreakable token)"
3739 );
3740 }
3741 let rejoined = out.join(" ");
3743 let original_words: Vec<&str> = line.split(' ').collect();
3744 let result_words: Vec<&str> = rejoined.split_whitespace().collect();
3745 assert_eq!(original_words, result_words, "reflow must preserve all words in order");
3746 }
3747
3748 #[test]
3753 fn test_helper_function_text_ends_with_abbreviation() {
3754 let abbreviations = get_abbreviations(&None);
3756
3757 assert!(text_ends_with_abbreviation("Dr.", &abbreviations));
3759 assert!(text_ends_with_abbreviation("word Dr.", &abbreviations));
3760 assert!(text_ends_with_abbreviation("e.g.", &abbreviations));
3761 assert!(text_ends_with_abbreviation("i.e.", &abbreviations));
3762 assert!(text_ends_with_abbreviation("Mr.", &abbreviations));
3763 assert!(text_ends_with_abbreviation("Mrs.", &abbreviations));
3764 assert!(text_ends_with_abbreviation("Ms.", &abbreviations));
3765 assert!(text_ends_with_abbreviation("Prof.", &abbreviations));
3766
3767 assert!(!text_ends_with_abbreviation("etc.", &abbreviations));
3769 assert!(!text_ends_with_abbreviation("paradigms.", &abbreviations));
3770 assert!(!text_ends_with_abbreviation("programs.", &abbreviations));
3771 assert!(!text_ends_with_abbreviation("items.", &abbreviations));
3772 assert!(!text_ends_with_abbreviation("systems.", &abbreviations));
3773 assert!(!text_ends_with_abbreviation("Dr?", &abbreviations)); assert!(!text_ends_with_abbreviation("Mr!", &abbreviations)); assert!(!text_ends_with_abbreviation("paradigms?", &abbreviations)); assert!(!text_ends_with_abbreviation("word", &abbreviations)); assert!(!text_ends_with_abbreviation("", &abbreviations)); }
3779
3780 #[test]
3781 fn test_footnote_after_period_splits_sentence() {
3782 let text = "First sentence.[^1] Second sentence.";
3786 let sentences = split_into_sentences(text);
3787 assert_eq!(
3788 sentences,
3789 vec!["First sentence.[^1]".to_string(), "Second sentence.".to_string()],
3790 "footnote glued to the period should keep the boundary and stay attached to the first sentence"
3791 );
3792 }
3793
3794 #[test]
3795 fn test_multiple_consecutive_footnotes_after_period_splits_sentence() {
3796 let text = "Notes here.[^1][^2] Second sentence.";
3798 let sentences = split_into_sentences(text);
3799 assert_eq!(
3800 sentences,
3801 vec!["Notes here.[^1][^2]".to_string(), "Second sentence.".to_string()]
3802 );
3803 }
3804
3805 #[test]
3806 fn test_footnote_before_period_still_splits_sentence() {
3807 let text = "Annotation here[^1]. Second sentence.";
3811 let sentences = split_into_sentences(text);
3812 assert_eq!(
3813 sentences,
3814 vec!["Annotation here[^1].".to_string(), "Second sentence.".to_string()]
3815 );
3816 }
3817
3818 #[test]
3819 fn test_mid_sentence_footnote_does_not_split() {
3820 let text = "The system word[^1] more words. Next sentence.";
3823 let sentences = split_into_sentences(text);
3824 assert_eq!(
3825 sentences,
3826 vec![
3827 "The system word[^1] more words.".to_string(),
3828 "Next sentence.".to_string()
3829 ]
3830 );
3831 }
3832
3833 #[test]
3834 fn test_bare_numeric_bracket_after_period_does_not_split() {
3835 let text = "Citation here.[1] Second sentence.";
3838 let sentences = split_into_sentences(text);
3839 assert_eq!(
3840 sentences,
3841 vec![text.to_string()],
3842 "a bare numeric bracket must not be treated as a sentence boundary"
3843 );
3844 }
3845
3846 #[test]
3847 fn test_footnote_glued_to_following_word_does_not_split() {
3848 let text = "First sentence.[^1]Continued glued text.";
3851 let sentences = split_into_sentences(text);
3852 assert_eq!(sentences, vec![text.to_string()]);
3853 }
3854
3855 #[test]
3856 fn test_footnote_at_end_of_text_is_preserved() {
3857 let text = "Sentence.[^1]";
3860 let sentences = split_into_sentences(text);
3861 assert_eq!(sentences, vec![text.to_string()]);
3862 }
3863
3864 #[test]
3865 fn test_abbreviation_before_footnote_does_not_split() {
3866 let text = "See the notes, e.g.[^1] this one.";
3869 let sentences = split_into_sentences(text);
3870 assert_eq!(
3871 sentences,
3872 vec![text.to_string()],
3873 "e.g. is an abbreviation, not a sentence boundary"
3874 );
3875 }
3876
3877 #[test]
3878 fn test_is_unordered_list_marker() {
3879 assert!(is_unordered_list_marker("- item"));
3881 assert!(is_unordered_list_marker("* item"));
3882 assert!(is_unordered_list_marker("+ item"));
3883 assert!(is_unordered_list_marker("-")); assert!(is_unordered_list_marker("*"));
3885 assert!(is_unordered_list_marker("+"));
3886
3887 assert!(!is_unordered_list_marker("---")); assert!(!is_unordered_list_marker("***")); assert!(!is_unordered_list_marker("- - -")); assert!(!is_unordered_list_marker("* * *")); assert!(!is_unordered_list_marker("*emphasis*")); assert!(!is_unordered_list_marker("-word")); assert!(!is_unordered_list_marker("")); assert!(!is_unordered_list_marker("text")); assert!(!is_unordered_list_marker("# heading")); }
3898
3899 #[test]
3900 fn test_is_block_boundary() {
3901 assert!(is_block_boundary("")); assert!(is_block_boundary("# Heading")); assert!(is_block_boundary("## Level 2")); assert!(is_block_boundary("```rust")); assert!(is_block_boundary("~~~")); assert!(is_block_boundary("> quote")); assert!(is_block_boundary("| cell |")); assert!(is_block_boundary("[link]: http://example.com")); assert!(is_block_boundary("---")); assert!(is_block_boundary("***")); assert!(is_block_boundary("- item")); assert!(is_block_boundary("* item")); assert!(is_block_boundary("+ item")); assert!(is_block_boundary("1. item")); assert!(is_block_boundary("10. item")); assert!(is_block_boundary(": definition")); assert!(is_block_boundary(":::")); assert!(is_block_boundary("::::: {.callout-note}")); assert!(!is_block_boundary("regular text"));
3923 assert!(!is_block_boundary("*emphasis*")); assert!(!is_block_boundary("[link](url)")); assert!(!is_block_boundary("some words here"));
3926 }
3927
3928 #[test]
3929 fn test_definition_list_boundary_in_single_line_paragraph() {
3930 let options = ReflowOptions {
3933 line_length: 80,
3934 ..Default::default()
3935 };
3936 let input = "Term\n: Definition of the term";
3937 let result = reflow_markdown(input, &options);
3938 assert!(
3940 result.contains(": Definition"),
3941 "Definition list item should not be merged into previous line. Got: {result:?}"
3942 );
3943 let lines: Vec<&str> = result.lines().collect();
3944 assert_eq!(lines.len(), 2, "Should remain two separate lines. Got: {lines:?}");
3945 assert_eq!(lines[0], "Term");
3946 assert_eq!(lines[1], ": Definition of the term");
3947 }
3948
3949 #[test]
3950 fn test_is_paragraph_boundary() {
3951 assert!(is_paragraph_boundary("# Heading", "# Heading"));
3953 assert!(is_paragraph_boundary("- item", "- item"));
3954 assert!(is_paragraph_boundary(":::", ":::"));
3955 assert!(is_paragraph_boundary(": definition", ": definition"));
3956
3957 assert!(is_paragraph_boundary("code", " code"));
3959 assert!(is_paragraph_boundary("code", "\tcode"));
3960
3961 assert!(is_paragraph_boundary("| a | b |", "| a | b |"));
3963 assert!(is_paragraph_boundary("a | b", "a | b")); assert!(!is_paragraph_boundary("regular text", "regular text"));
3967 assert!(!is_paragraph_boundary("text", " text")); }
3969
3970 #[test]
3971 fn test_div_marker_boundary_in_reflow_paragraph_at_line() {
3972 let content = "Some paragraph text here.\n\n::: {.callout-note}\nThis is a callout.\n:::\n";
3975 let result = reflow_paragraph_at_line(content, 3, 80);
3977 assert!(result.is_none(), "Div marker line should not be reflowed");
3978 }
3979
3980 #[test]
3981 fn starts_block_construct_detects_block_openers() {
3982 for case in ["- item", "-", "* item", "*", "+ item", "+", "-\titem"] {
3984 assert!(starts_block_construct(case), "bullet: {case:?}");
3985 }
3986 for case in ["1. item", "1) item", "9. x", "123456789. x", "1.", "42) x"] {
3988 assert!(starts_block_construct(case), "ordered: {case:?}");
3989 }
3990 for case in ["> quote", ">quote", ">"] {
3992 assert!(starts_block_construct(case), "blockquote: {case:?}");
3993 }
3994 for case in ["# heading", "###### h6", "#", "##"] {
3996 assert!(starts_block_construct(case), "heading: {case:?}");
3997 }
3998 for case in ["```", "```rust", "````", "~~~", "~~~text"] {
4000 assert!(starts_block_construct(case), "fence: {case:?}");
4001 }
4002 for case in ["---", "--", "===", "=", "***", "___", "_ _ _", "- - -"] {
4004 assert!(starts_block_construct(case), "setext/thematic: {case:?}");
4005 }
4006 for case in [
4009 "[^1]: text",
4010 "[^note]:",
4011 "[ref]: http://example.com",
4012 "[wat]: url follows",
4013 ] {
4014 assert!(starts_block_construct(case), "definition: {case:?}");
4015 }
4016 for case in ["<div>content", "</div>", "<p>text", "<table>", "<pre>code", "<h1>x"] {
4018 assert!(starts_block_construct(case), "html block: {case:?}");
4019 }
4020 }
4021
4022 #[test]
4023 fn starts_block_construct_allows_ordinary_prose() {
4024 for case in [
4025 "",
4026 "word",
4027 "-5 degrees",
4028 "--flag",
4029 "-item",
4030 "#hashtag",
4031 "####### seven hashes is not a heading",
4032 "1.5 million",
4033 "1234567890. ten digits is not a list marker",
4034 "1:30 pm",
4035 "*emphasis*",
4036 "**bold** text",
4037 "__bold__ text",
4038 "_emphasis_ text",
4039 "`code` span",
4040 "`` double backtick span ``",
4041 "~~strikethrough~~",
4042 "=x",
4043 "== ==",
4044 "(parenthetical)",
4045 "[link](url)",
4046 "[text][ref] more",
4047 "[bracketed] aside",
4048 "[a](b) [ref]: first bracket is a link, not a label",
4049 "[esc\\]: not a close] text",
4050 "<span>inline</span>",
4051 "<b>bold</b>",
4052 "<https://example.com> autolink",
4053 "<mailto:a@b.com>",
4054 "<notarealtag>",
4055 ] {
4056 assert!(!starts_block_construct(case), "prose: {case:?}");
4057 }
4058 }
4059
4060 #[test]
4061 fn merge_block_construct_continuations_merges_marker_led_lines() {
4062 let lines = vec![
4063 "First sentence?".to_string(),
4064 "- looks like a list item".to_string(),
4065 "Second sentence.".to_string(),
4066 ];
4067 assert_eq!(
4068 merge_block_construct_continuations(lines),
4069 vec![
4070 "First sentence? - looks like a list item".to_string(),
4071 "Second sentence.".to_string(),
4072 ]
4073 );
4074
4075 let lines = vec!["- real list content".to_string(), "continuation".to_string()];
4078 assert_eq!(
4079 merge_block_construct_continuations(lines.clone()),
4080 lines,
4081 "first line must never be merged"
4082 );
4083 }
4084
4085 #[test]
4086 fn wrap_never_starts_a_line_with_a_block_marker() {
4087 let options = ReflowOptions {
4088 line_length: 25,
4089 ..Default::default()
4090 };
4091 let lines = reflow_line(
4094 "Some words here and then - a dash clause that wraps around the limit.",
4095 &options,
4096 );
4097 assert_eq!(
4098 lines,
4099 vec![
4100 "Some words here and",
4101 "then - a dash clause that",
4102 "wraps around the limit."
4103 ]
4104 );
4105
4106 for input in [
4108 "Alpha beta gamma delta epsilon - dash clause here to wrap",
4109 "Alpha beta gamma delta epsilon > quote lookalike here to wrap",
4110 "Alpha beta gamma delta epsilon # heading lookalike here to wrap",
4111 "Alpha beta gamma delta epsilon 1. ordered lookalike here to wrap",
4112 "Alpha beta gamma delta epsilon * star clause here to wrap",
4113 "Alpha beta gamma delta epsilon + plus clause here to wrap",
4114 ] {
4115 for width in 10..40 {
4116 let options = ReflowOptions {
4117 line_length: width,
4118 ..Default::default()
4119 };
4120 for line in reflow_line(input, &options) {
4121 assert!(
4122 !starts_block_construct(&line),
4123 "width {width}: wrapped line opens a block construct: {line:?} (input {input:?})"
4124 );
4125 }
4126 }
4127 }
4128 }
4129
4130 #[test]
4131 fn sentence_per_line_keeps_block_markers_mid_line() {
4132 let options = ReflowOptions {
4133 line_length: 80,
4134 sentence_per_line: true,
4135 ..Default::default()
4136 };
4137 let lines = reflow_line(
4140 "Google Calendar (Can't we get rid of this dependency? - I don't really see the need)",
4141 &options,
4142 );
4143 assert_eq!(
4144 lines,
4145 vec!["Google Calendar (Can't we get rid of this dependency? - I don't really see the need)".to_string()]
4146 );
4147
4148 let lines = reflow_line("See section 4? # is the marker we use. Fine.", &options);
4150 assert_eq!(lines, vec!["See section 4? # is the marker we use.", "Fine."]);
4151
4152 let lines = reflow_line("Is this a problem? > I quote someone here.", &options);
4153 assert_eq!(lines, vec!["Is this a problem? > I quote someone here."]);
4154
4155 let lines = reflow_line("Another case! 1. Not a list. More text follows here.", &options);
4156 for line in &lines {
4157 assert!(
4158 !starts_block_construct(line),
4159 "sentence-per-line output opens a block construct: {line:?}"
4160 );
4161 }
4162 }
4163
4164 #[test]
4165 fn inline_math_directly_after_display_math_stays_atomic() {
4166 let options = ReflowOptions {
4174 line_length: 8,
4175 ..Default::default()
4176 };
4177 let lines = reflow_line("$$a$$$bb cc dd$ x", &options);
4178 assert_eq!(lines, vec!["$$a$$$bb cc dd$".to_string(), "x".to_string()]);
4179 }
4180
4181 #[test]
4182 fn test_code_span_parsing() {
4183 let elements = parse_markdown_elements_inner("`code`", false, false, None);
4185 assert_eq!(elements.len(), 1);
4186 assert!(matches!(&elements[0], Element::Code(s) if s == "`code`"));
4187
4188 let elements = parse_markdown_elements_inner("``code``", false, false, None);
4190 assert_eq!(elements.len(), 1);
4191 assert!(matches!(&elements[0], Element::Code(s) if s == "``code``"));
4192
4193 let elements = parse_markdown_elements_inner("``code`inside``", false, false, None);
4195 assert_eq!(elements.len(), 1);
4196 assert!(matches!(&elements[0], Element::Code(s) if s == "``code`inside``"));
4197
4198 let elements = parse_markdown_elements_inner("`` code ``", false, false, None);
4200 assert_eq!(elements.len(), 1);
4201 assert!(matches!(&elements[0], Element::Code(s) if s == "`` code ``"));
4202
4203 let elements = parse_markdown_elements_inner("`unclosed", false, false, None);
4205 assert_eq!(elements.len(), 1);
4206 assert!(matches!(&elements[0], Element::Text(s) if s == "`unclosed"));
4207
4208 let elements = parse_markdown_elements_inner("`unclosed [link](url)", false, false, None);
4210 assert_eq!(elements.len(), 2);
4212 assert!(matches!(&elements[0], Element::Text(s) if s == "`unclosed "));
4213 assert!(matches!(&elements[1], Element::Link(s) if s == "[link](url)"));
4214 }
4215
4216 #[test]
4217 fn test_reflow_performance_long_input() {
4218 let mut text = String::new();
4221 for i in 1..400 {
4222 let backticks = "`".repeat(i);
4223 text.push_str(&backticks);
4224 text.push(' ');
4225 }
4226
4227 let start = std::time::Instant::now();
4228 let elements = parse_markdown_elements_inner(&text, false, false, None);
4229 let duration = start.elapsed();
4230
4231 assert!(duration.as_millis() < 100, "Parsing took too long: {duration:?}");
4233 assert!(!elements.is_empty());
4234 }
4235
4236 #[test]
4237 fn test_reflow_performance_display_math_heavy() {
4238 let text = "$$a$$".repeat(4000);
4243
4244 let start = std::time::Instant::now();
4245 let elements = parse_markdown_elements_inner(&text, false, false, None);
4246 let duration = start.elapsed();
4247
4248 assert!(duration.as_millis() < 100, "Parsing took too long: {duration:?}");
4249 assert_eq!(elements.len(), 4000);
4250 }
4251
4252 #[test]
4253 fn inline_math_len_at_start_matches_regex_at_slice_start() {
4254 let alphabet = ['$', 'a', ' '];
4259 let mut inputs: Vec<String> = vec![String::new()];
4260 let mut frontier: Vec<String> = vec![String::new()];
4261 for _ in 0..6 {
4262 let mut longer = Vec::new();
4263 for prefix in &frontier {
4264 for ch in alphabet {
4265 let mut s = prefix.clone();
4266 s.push(ch);
4267 longer.push(s);
4268 }
4269 }
4270 inputs.extend(longer.iter().cloned());
4271 frontier = longer;
4272 }
4273 inputs.push("$αβ$x".to_string());
4275 inputs.push("$α$$".to_string());
4276
4277 for s in &inputs {
4278 let expected = INLINE_MATH_REGEX
4279 .find(s)
4280 .ok()
4281 .flatten()
4282 .filter(|m| m.start() == 0)
4283 .map(|m| m.end());
4284 assert_eq!(inline_math_len_at_start(s), expected, "input: {s:?}");
4285 }
4286 }
4287
4288 #[test]
4289 fn inline_math_probe_after_dollar_matches_uncached_parse() {
4290 let cases = [
4296 ("$$a$$$b c$ x", r#"[DisplayMath("a"), InlineMath("b c"), Text(" x")]"#),
4297 (
4298 "$$a$$$b$ $$a$$$b$",
4299 r#"[DisplayMath("a"), InlineMath("b"), Text(" "), DisplayMath("a"), InlineMath("b")]"#,
4300 ),
4301 (
4303 "$$a$$$ x $y z$",
4304 r#"[DisplayMath("a"), InlineMath(" x "), Text("y z$")]"#,
4305 ),
4306 ("$$a$$$$ x", r#"[DisplayMath("a"), Text("$$ x")]"#),
4308 ("$$a$$$$b$$ x", r#"[DisplayMath("a"), DisplayMath("b"), Text(" x")]"#),
4309 (
4311 "$a$$b$$c$$d$ tail",
4312 r#"[Text("$a"), DisplayMath("b"), Text("c$$d$ tail")]"#,
4313 ),
4314 ];
4315 for (input, expected) in cases {
4316 let elements = parse_markdown_elements_inner(input, false, false, None);
4317 assert_eq!(format!("{elements:?}"), expected, "input: {input:?}");
4318 }
4319 }
4320}