use crate::utils::calculate_indentation_width_default;
use crate::utils::is_definition_list_item;
use crate::utils::mkdocs_attr_list::{ATTR_LIST_PATTERN, is_standalone_attr_list};
use crate::utils::mkdocs_snippets::is_snippet_block_delimiter;
use crate::utils::regex_cache::{
DISPLAY_MATH_REGEX, EMAIL_PATTERN, EMOJI_SHORTCODE_REGEX, HTML_ENTITY_REGEX, HTML_TAG_PATTERN,
HUGO_SHORTCODE_REGEX, INLINE_MATH_REGEX, WIKI_LINK_REGEX,
};
use crate::utils::sentence_utils::{
get_abbreviations, is_cjk_char, is_cjk_sentence_ending, is_closing_quote, is_opening_quote,
text_ends_with_abbreviation,
};
use pulldown_cmark::{BrokenLink, CowStr, Event, LinkType, Options, Parser, Tag, TagEnd};
use std::collections::HashSet;
use unicode_width::UnicodeWidthStr;
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub enum ReflowLengthMode {
Chars,
#[default]
Visual,
Bytes,
}
fn display_len(s: &str, mode: ReflowLengthMode) -> usize {
match mode {
ReflowLengthMode::Chars => s.chars().count(),
ReflowLengthMode::Visual => s.width(),
ReflowLengthMode::Bytes => s.len(),
}
}
fn is_non_breaking_space(c: char) -> bool {
matches!(c, '\u{00A0}' | '\u{202F}' | '\u{2007}')
}
fn is_breakable_whitespace(c: char) -> bool {
c.is_whitespace() && !is_non_breaking_space(c)
}
fn split_breakable_words(text: &str) -> impl Iterator<Item = &str> {
text.split(is_breakable_whitespace).filter(|word| !word.is_empty())
}
fn code_span_wraps_losslessly(content: &str) -> bool {
let mut prev_ws = false;
for c in content.chars() {
let ws = is_breakable_whitespace(c);
if ws && (prev_ws || c != ' ') {
return false;
}
prev_ws = ws;
}
true
}
struct NestedStructure {
atomic: Vec<(usize, usize)>,
markers: Vec<(usize, usize)>,
links: Vec<(usize, usize)>,
}
struct OpenSpan {
span: (usize, usize),
content: Option<(usize, usize)>,
}
fn note_span_content(open: &mut [OpenSpan], start: usize, end: usize) {
for open_span in open.iter_mut() {
if start >= open_span.span.0 && end <= open_span.span.1 {
open_span.content = Some(match open_span.content {
Some((known_start, known_end)) => (known_start.min(start), known_end.max(end)),
None => (start, end),
});
}
}
}
fn merge_ranges(mut ranges: Vec<(usize, usize)>) -> Vec<(usize, usize)> {
ranges.sort_unstable();
let mut merged: Vec<(usize, usize)> = Vec::with_capacity(ranges.len());
for (start, end) in ranges {
match merged.last_mut() {
Some(last) if start <= last.1 => last.1 = last.1.max(end),
_ => merged.push((start, end)),
}
}
merged
}
fn nested_structure(content: &str, defined_references: Option<&HashSet<String>>, attr_lists: bool) -> NestedStructure {
let mut options = Options::empty();
options.insert(Options::ENABLE_STRIKETHROUGH);
let mut atomic: Vec<(usize, usize)> = Vec::new();
let mut markers: Vec<(usize, usize)> = Vec::new();
let mut links: Vec<(usize, usize)> = Vec::new();
let mut open: Vec<OpenSpan> = Vec::new();
for (event, range) in Parser::new_ext(content, options).into_offset_iter() {
let (start, end) = (range.start, range.end);
if !matches!(event, Event::End(_)) {
note_span_content(&mut open, start, end);
}
match event {
Event::Start(Tag::Link { .. } | Tag::Image { .. }) => {
atomic.push((start, end));
links.push((start, end));
}
Event::Code(_) | Event::InlineHtml(_) => {
atomic.push((start, end));
}
Event::Start(Tag::Emphasis | Tag::Strong | Tag::Strikethrough) => {
open.push(OpenSpan {
span: (start, end),
content: None,
});
}
Event::End(TagEnd::Emphasis | TagEnd::Strong | TagEnd::Strikethrough) => {
if let Some(OpenSpan {
span: (span_start, span_end),
content,
}) = open.pop()
{
match content {
Some((content_start, content_end)) => {
markers.push((span_start, content_start));
markers.push((content_end, span_end));
}
None => atomic.push((span_start, span_end)),
}
}
}
_ => {}
}
}
for span in all_link_spans(content, defined_references) {
atomic.push((span.start, span.end));
links.push((span.start, span.end));
}
for found in WIKI_LINK_REGEX.find_iter(content) {
atomic.push((found.start(), found.end()));
links.push((found.start(), found.end()));
}
for found in HUGO_SHORTCODE_REGEX
.find_iter(content)
.chain(DISPLAY_MATH_REGEX.find_iter(content))
{
atomic.push((found.start(), found.end()));
}
let mut from = 0;
while let Ok(Some(found)) = INLINE_MATH_REGEX.find_from_pos(content, from) {
atomic.push((found.start(), found.end()));
from = found.end();
}
if attr_lists {
for found in ATTR_LIST_PATTERN.find_iter(content) {
atomic.push((found.start(), found.end()));
}
}
links.sort_unstable();
links.dedup();
NestedStructure {
atomic: merge_ranges(atomic),
markers: merge_ranges(markers),
links,
}
}
fn breakable_units<'a>(
content: &'a str,
defined_references: Option<&HashSet<String>>,
attr_lists: bool,
) -> Option<Vec<&'a str>> {
if !content.contains(['`', '*', '_', '~', '[', '<', '$', '{']) {
return Some(split_breakable_words(content).collect());
}
let NestedStructure { atomic, markers, .. } = nested_structure(content, defined_references, attr_lists);
let mut units = Vec::new();
let mut unit_start = None;
let mut next_atomic = 0;
let mut next_marker = 0;
for (offset, ch) in content.char_indices() {
while atomic.get(next_atomic).is_some_and(|&(_, end)| end <= offset) {
next_atomic += 1;
}
if atomic.get(next_atomic).is_some_and(|&(start, _)| offset >= start) {
if unit_start.is_none() {
unit_start = Some(offset);
}
continue;
}
while markers.get(next_marker).is_some_and(|&(_, end)| end <= offset) {
next_marker += 1;
}
if matches!(ch, '`' | '*' | '_' | '~') && markers.get(next_marker).is_none_or(|&(start, _)| offset < start) {
return None;
}
if is_breakable_whitespace(ch) {
if let Some(start) = unit_start.take() {
units.push(&content[start..offset]);
}
} else if unit_start.is_none() {
unit_start = Some(offset);
}
}
if let Some(start) = unit_start {
units.push(&content[start..]);
}
Some(units)
}
#[derive(Clone)]
pub struct ReflowOptions {
pub line_length: usize,
pub break_on_sentences: bool,
pub preserve_breaks: bool,
pub sentence_per_line: bool,
pub semantic_line_breaks: bool,
pub abbreviations: Option<Vec<String>>,
pub length_mode: ReflowLengthMode,
pub attr_lists: bool,
pub myst_roles: bool,
pub require_sentence_capital: bool,
pub max_list_continuation_indent: Option<usize>,
pub defined_references: Option<HashSet<String>>,
pub atomic_spans: bool,
pub length_exemptions: LengthExemptions,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct LengthExemptions {
pub link_urls: bool,
pub code_spans: bool,
}
impl LengthExemptions {
fn any(&self) -> bool {
self.link_urls || self.code_spans
}
}
impl Default for ReflowOptions {
fn default() -> Self {
Self {
line_length: 80,
break_on_sentences: true,
preserve_breaks: false,
sentence_per_line: false,
semantic_line_breaks: false,
abbreviations: None,
length_mode: ReflowLengthMode::default(),
attr_lists: false,
myst_roles: false,
require_sentence_capital: true,
max_list_continuation_indent: None,
defined_references: None,
atomic_spans: true,
length_exemptions: LengthExemptions::default(),
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct LineWidth {
link_exempt: usize,
code_exempt: usize,
}
impl LineWidth {
fn plain(width: usize) -> Self {
Self {
link_exempt: width,
code_exempt: width,
}
}
fn effective(self) -> usize {
self.link_exempt.min(self.code_exempt)
}
fn fits(self, line_length: usize) -> bool {
self.effective() <= line_length
}
fn is_empty(self) -> bool {
self.link_exempt == 0 && self.code_exempt == 0
}
}
impl std::ops::Add for LineWidth {
type Output = Self;
fn add(self, other: Self) -> Self {
Self {
link_exempt: self.link_exempt + other.link_exempt,
code_exempt: self.code_exempt + other.code_exempt,
}
}
}
impl std::ops::AddAssign for LineWidth {
fn add_assign(&mut self, other: Self) {
*self = *self + other;
}
}
pub fn normalize_reference_label(label: &str) -> String {
label.split_whitespace().collect::<Vec<_>>().join(" ").to_lowercase()
}
fn footnote_refs_end(chars: &[char], start: usize) -> Option<usize> {
let mut pos = start;
let mut found = false;
loop {
if chars.get(pos) != Some(&'[') || chars.get(pos + 1) != Some(&'^') {
break;
}
let label_start = pos + 2;
let mut label_end = label_start;
while matches!(chars.get(label_end), Some(c) if c.is_ascii_alphanumeric() || *c == '_' || *c == '-') {
label_end += 1;
}
if label_end == label_start || chars.get(label_end) != Some(&']') {
break;
}
pos = label_end + 1;
found = true;
}
found.then_some(pos)
}
fn char_byte_offsets(chars: &[char]) -> Vec<usize> {
let mut offsets = Vec::with_capacity(chars.len() + 1);
let mut offset = 0;
for c in chars {
offsets.push(offset);
offset += c.len_utf8();
}
offsets.push(offset);
offsets
}
struct SentenceText<'a> {
text: &'a str,
chars: &'a [char],
char_offsets: &'a [usize],
links: &'a [(usize, usize)],
}
impl SentenceText<'_> {
fn link_end_at(&self, pos: usize) -> Option<usize> {
let range_start = match self.chars.get(pos) {
Some('[') => pos,
Some('!') if self.chars.get(pos + 1) == Some(&'[') => match self.link_range_end_at(pos) {
Some(end) => return Some(end),
None => pos + 1,
},
_ => return None,
};
self.link_range_end_at(range_start)
}
fn link_range_end_at(&self, pos: usize) -> Option<usize> {
let start = self.char_offsets[pos];
let idx = self.links.binary_search_by_key(&start, |&(s, _)| s).ok()?;
let end = self.links[idx].1;
Some(self.char_offsets.binary_search(&end).unwrap_or_else(|i| i))
}
}
fn is_sentence_boundary(
st: &SentenceText<'_>,
pos: usize,
abbreviations: &HashSet<String>,
require_sentence_capital: bool,
) -> bool {
let SentenceText { text, chars, .. } = *st;
if pos + 1 >= chars.len() {
return false;
}
let byte_offset_after_punct = st.char_offsets[pos + 1];
let c = chars[pos];
let next_char = chars[pos + 1];
if is_cjk_sentence_ending(c) {
let mut after_punct_pos = pos + 1;
while after_punct_pos < chars.len()
&& (chars[after_punct_pos] == '*' || chars[after_punct_pos] == '_' || chars[after_punct_pos] == '~')
{
after_punct_pos += 1;
}
while after_punct_pos < chars.len() && chars[after_punct_pos].is_whitespace() {
after_punct_pos += 1;
}
if after_punct_pos >= chars.len() {
return false;
}
if opens_ordered_list_marker(&chars[after_punct_pos..]) {
return false;
}
while after_punct_pos < chars.len()
&& (chars[after_punct_pos] == '*' || chars[after_punct_pos] == '_' || chars[after_punct_pos] == '~')
{
after_punct_pos += 1;
}
if after_punct_pos >= chars.len() {
return false;
}
return true;
}
if c != '.' && c != '!' && c != '?' {
return false;
}
let inside_quotation = is_closing_quote(next_char);
let (_space_pos, after_space_pos) = if next_char == ' ' {
(pos + 1, pos + 2)
} else if is_closing_quote(next_char) && pos + 2 < chars.len() {
if chars[pos + 2] == ' ' {
(pos + 2, pos + 3)
} else if (chars[pos + 2] == '*' || chars[pos + 2] == '_') && pos + 3 < chars.len() && chars[pos + 3] == ' ' {
(pos + 3, pos + 4)
} else if (chars[pos + 2] == '*' || chars[pos + 2] == '_')
&& pos + 4 < chars.len()
&& chars[pos + 3] == chars[pos + 2]
&& chars[pos + 4] == ' '
{
(pos + 4, pos + 5)
} else {
return false;
}
} else if (next_char == '*' || next_char == '_') && pos + 2 < chars.len() && chars[pos + 2] == ' ' {
(pos + 2, pos + 3)
} else if (next_char == '*' || next_char == '_')
&& pos + 3 < chars.len()
&& chars[pos + 2] == next_char
&& chars[pos + 3] == ' '
{
(pos + 3, pos + 4)
} else if next_char == '~' && pos + 3 < chars.len() && chars[pos + 2] == '~' && chars[pos + 3] == ' ' {
(pos + 3, pos + 4)
} else if next_char == '[' {
match footnote_refs_end(chars, pos + 1) {
Some(end_pos) if chars.get(end_pos) == Some(&' ') => (end_pos, end_pos + 1),
_ => return false,
}
} else {
return false;
};
let mut next_char_pos = after_space_pos;
while next_char_pos < chars.len() && chars[next_char_pos].is_whitespace() {
next_char_pos += 1;
}
if next_char_pos >= chars.len() {
return false;
}
if opens_ordered_list_marker(&chars[next_char_pos..]) {
return false;
}
let mut first_letter_pos = next_char_pos;
while first_letter_pos < chars.len() {
let ch = chars[first_letter_pos];
if let Some(end) = st.link_end_at(first_letter_pos) {
first_letter_pos += link_opener_len(chars, first_letter_pos, end);
} else if matches!(ch, '*' | '_' | '~') || is_opening_quote(ch) {
first_letter_pos += 1;
} else {
break;
}
}
if first_letter_pos >= chars.len() {
return false;
}
let first_char = chars[first_letter_pos];
if c == '!' || c == '?' {
return !inside_quotation || !require_sentence_capital || opens_sentence_in_strict_mode(first_char);
}
if pos > 0 {
if text_ends_with_abbreviation(&text[..byte_offset_after_punct], abbreviations) {
return false;
}
if chars[pos - 1].is_ascii_uppercase() && (pos == 1 || (pos >= 2 && chars[pos - 2].is_whitespace())) {
return false;
}
}
if require_sentence_capital && !opens_sentence_in_strict_mode(first_char) {
return false;
}
true
}
fn opens_sentence_in_strict_mode(first_char: char) -> bool {
first_char.is_uppercase() || first_char.is_numeric() || is_cjk_char(first_char)
}
fn opens_ordered_list_marker(chars: &[char]) -> bool {
let digits = chars.iter().take_while(|c| c.is_ascii_digit()).count();
digits > 0 && matches!(chars.get(digits), Some('.' | ')')) && matches!(chars.get(digits + 1), Some(' ' | '\t'))
}
fn link_opener_len(chars: &[char], pos: usize, end: usize) -> usize {
let open = if chars[pos] == '!' { pos + 1 } else { pos };
let body = open + 1;
if chars.get(body) != Some(&'[') {
return body - pos;
}
let body = body + 1;
let alias = chars[body..end.saturating_sub(2).max(body)]
.iter()
.position(|&c| c == '|')
.map_or(body, |p| body + p + 1);
alias - pos
}
pub fn split_into_sentences(text: &str, defined_references: Option<&HashSet<String>>) -> Vec<String> {
let abbreviations = get_abbreviations(&None);
split_into_sentences_with_set(text, &abbreviations, true, None, defined_references)
}
fn split_into_sentences_with_set(
text: &str,
abbreviations: &HashSet<String>,
require_sentence_capital: bool,
appended_span_start: Option<usize>,
defined_references: Option<&HashSet<String>>,
) -> Vec<String> {
let char_vec: Vec<char> = text.chars().collect();
let char_offsets = char_byte_offsets(&char_vec);
let NestedStructure { atomic, links, .. } = sentence_structure(text, defined_references);
let mut atomic_it = atomic.iter().peekable();
let st = SentenceText {
text,
chars: &char_vec,
char_offsets: &char_offsets,
links: &links,
};
let mut sentences = Vec::new();
let mut current_sentence = String::new();
let mut pos = 0;
while pos < char_vec.len() {
let c = char_vec[pos];
current_sentence.push(c);
let byte_idx = char_offsets[pos];
while let Some(&&(_, end)) = atomic_it.peek() {
if end <= byte_idx {
atomic_it.next();
} else {
break;
}
}
let in_atomic = atomic_it
.peek()
.is_some_and(|&&(start, end)| byte_idx >= start && byte_idx < end);
if !in_atomic && is_sentence_boundary(&st, pos, abbreviations, require_sentence_capital) {
if let Some(end_pos) = footnote_refs_end(&char_vec, pos + 1) {
while pos + 1 < end_pos {
pos += 1;
current_sentence.push(char_vec[pos]);
}
}
while pos + 1 < char_vec.len() {
let next = char_vec[pos + 1];
if matches!(next, '*' | '_' | '~') && Some(char_offsets[pos + 1]) == appended_span_start {
break;
}
if next == '*' || next == '_' || next == '~' || is_closing_quote(next) {
pos += 1;
current_sentence.push(char_vec[pos]);
} else {
break;
}
}
if pos + 1 < char_vec.len() && char_vec[pos + 1] == ' ' {
pos += 1; }
sentences.push(current_sentence.trim().to_string());
current_sentence.clear();
}
pos += 1;
}
if !current_sentence.trim().is_empty() {
sentences.push(current_sentence.trim().to_string());
}
sentences
}
fn sentence_structure(text: &str, defined_references: Option<&HashSet<String>>) -> NestedStructure {
if !text.contains(['`', '[', '<', '$']) {
return NestedStructure {
atomic: Vec::new(),
markers: Vec::new(),
links: Vec::new(),
};
}
nested_structure(text, defined_references, false)
}
fn is_horizontal_rule(line: &str) -> bool {
if line.len() < 3 {
return false;
}
let mut chars = line.chars();
let Some(first_char) = chars.next() else {
return false;
};
if first_char != '-' && first_char != '_' && first_char != '*' {
return false;
}
let mut non_space_count = 1usize; for c in chars {
if c == ' ' {
continue;
}
if c != first_char {
return false;
}
non_space_count += 1;
}
non_space_count >= 3
}
fn is_numbered_list_item(line: &str) -> bool {
let mut chars = line.chars();
if !chars.next().is_some_and(char::is_numeric) {
return false;
}
while let Some(c) = chars.next() {
if c == '.' {
return chars.next() == Some(' ');
}
if !c.is_numeric() {
return false;
}
}
false
}
fn is_unordered_list_marker(s: &str) -> bool {
matches!(s.as_bytes().first(), Some(b'-' | b'*' | b'+'))
&& !is_horizontal_rule(s)
&& (s.len() == 1 || s.as_bytes().get(1) == Some(&b' '))
}
fn is_block_boundary_core(trimmed: &str) -> bool {
trimmed.is_empty()
|| trimmed.starts_with('#')
|| trimmed.starts_with("```")
|| trimmed.starts_with("~~~")
|| trimmed.starts_with('>')
|| (trimmed.starts_with('[') && trimmed.contains("]:"))
|| is_horizontal_rule(trimmed)
|| is_unordered_list_marker(trimmed)
|| is_numbered_list_item(trimmed)
|| is_definition_list_item(trimmed)
|| trimmed.starts_with(":::")
}
fn is_block_boundary(trimmed: &str) -> bool {
is_block_boundary_core(trimmed) || trimmed.starts_with('|')
}
fn is_paragraph_boundary(trimmed: &str, line: &str) -> bool {
is_block_boundary_core(trimmed)
|| calculate_indentation_width_default(line) >= 4
|| crate::utils::table_utils::TableUtils::is_potential_table_row(line)
}
fn has_hard_break(line: &str) -> bool {
let line = line.strip_suffix('\r').unwrap_or(line);
line.ends_with(" ") || line.ends_with('\\')
}
fn ends_with_sentence_punct(text: &str) -> bool {
text.ends_with('.') || text.ends_with('!') || text.ends_with('?')
}
fn trim_preserving_hard_break(s: &str) -> String {
let s = s.strip_suffix('\r').unwrap_or(s);
if s.ends_with('\\') {
return s.to_string();
}
if s.ends_with(" ") {
let content_end = s.trim_end().len();
if content_end == 0 {
return String::new();
}
format!("{} ", &s[..content_end])
} else {
s.trim_end().to_string()
}
}
fn parse_elements(text: &str, options: &ReflowOptions) -> Vec<Element> {
parse_markdown_elements_inner(
text,
options.attr_lists,
options.myst_roles,
options.defined_references.as_ref(),
)
}
pub fn reflow_line(line: &str, options: &ReflowOptions) -> Vec<String> {
let reflowed = reflow_line_unchecked(line, options);
if preserves_content(line, &reflowed) {
reflowed
} else {
vec![line.to_string()]
}
}
fn preserves_content(original: &str, reflowed: &[String]) -> bool {
let (original_text, original_breaks) = visible_text_and_breaks(original.chars());
let (reflowed_text, reflowed_breaks) =
visible_text_and_breaks(reflowed.iter().flat_map(|line| line.chars().chain(['\n'])));
original_text == reflowed_text && contains_all(&reflowed_breaks, &original_breaks)
}
fn visible_text_and_breaks(text: impl Iterator<Item = char>) -> (String, Vec<usize>) {
let mut visible = String::new();
let mut breaks = Vec::new();
let mut count = 0usize;
let mut pending_break = false;
for c in text {
if c.is_whitespace() {
pending_break = count > 0;
} else {
if pending_break {
breaks.push(count);
pending_break = false;
}
visible.push(c);
count += 1;
}
}
(visible, breaks)
}
fn contains_all(superset: &[usize], subset: &[usize]) -> bool {
let mut candidates = superset.iter();
subset
.iter()
.all(|wanted| candidates.by_ref().any(|found| found == wanted))
}
fn reflow_line_unchecked(line: &str, options: &ReflowOptions) -> Vec<String> {
if options.sentence_per_line {
let elements = parse_elements(line, options);
return merge_block_construct_continuations(reflow_elements_sentence_per_line(&elements, options));
}
if options.semantic_line_breaks {
let elements = parse_elements(line, options);
return merge_block_construct_continuations(reflow_elements_semantic(&elements, options));
}
if options.line_length == 0 || line_fits(line, options) {
return vec![line.to_string()];
}
let elements = parse_elements(line, options);
merge_block_construct_continuations(reflow_elements(&elements, options))
}
#[derive(Debug, Clone)]
enum Element {
Text(String),
Link(String),
ReferenceLink(String),
EmptyReferenceLink(String),
ShortcutReference(String),
InlineImage(String),
ReferenceImage(String),
EmptyReferenceImage(String),
LinkedImage(String),
FootnoteReference(String),
Strikethrough {
content: String,
double: bool,
},
WikiLink(String),
InlineMath(String),
DisplayMath(String),
EmojiShortcode(String),
Autolink(String),
HtmlTag(String),
HtmlEntity(String),
HugoShortcode(String),
AttrList(String),
MystRole(String),
Code { content: String, marker: String },
Bold {
content: String,
underscore: bool,
},
Italic {
content: String,
underscore: bool,
},
}
impl Element {
fn opens_with_bracket(&self) -> bool {
matches!(
self,
Element::Link(_)
| Element::ReferenceLink(_)
| Element::EmptyReferenceLink(_)
| Element::ShortcutReference(_)
| Element::FootnoteReference(_)
| Element::InlineImage(_)
| Element::ReferenceImage(_)
| Element::EmptyReferenceImage(_)
| Element::LinkedImage(_)
| Element::WikiLink(_)
)
}
}
impl std::fmt::Display for Element {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Element::Text(s) => write!(f, "{s}"),
Element::Link(s) => write!(f, "{s}"),
Element::ReferenceLink(s) => write!(f, "{s}"),
Element::EmptyReferenceLink(s) => write!(f, "{s}"),
Element::ShortcutReference(s) => write!(f, "{s}"),
Element::InlineImage(s) => write!(f, "{s}"),
Element::ReferenceImage(s) => write!(f, "{s}"),
Element::EmptyReferenceImage(s) => write!(f, "{s}"),
Element::LinkedImage(s) => write!(f, "{s}"),
Element::FootnoteReference(s) => write!(f, "{s}"),
Element::Strikethrough { content, double } => {
let marker = if *double { "~~" } else { "~" };
write!(f, "{marker}{content}{marker}")
}
Element::WikiLink(s) => write!(f, "[[{s}]]"),
Element::InlineMath(s) => write!(f, "${s}$"),
Element::DisplayMath(s) => write!(f, "$${s}$$"),
Element::EmojiShortcode(s) => write!(f, ":{s}:"),
Element::Autolink(s) => write!(f, "{s}"),
Element::HtmlTag(s) => write!(f, "{s}"),
Element::HtmlEntity(s) => write!(f, "{s}"),
Element::HugoShortcode(s) => write!(f, "{s}"),
Element::AttrList(s) => write!(f, "{s}"),
Element::MystRole(s) => write!(f, "{s}"),
Element::Code { content, marker } => write!(f, "{marker}{content}{marker}"),
Element::Bold { content, underscore } => {
if *underscore {
write!(f, "__{content}__")
} else {
write!(f, "**{content}**")
}
}
Element::Italic { content, underscore } => {
if *underscore {
write!(f, "_{content}_")
} else {
write!(f, "*{content}*")
}
}
}
}
}
impl Element {
fn display_len(&self, mode: ReflowLengthMode) -> usize {
match self {
Element::Text(s)
| Element::Link(s)
| Element::ReferenceLink(s)
| Element::EmptyReferenceLink(s)
| Element::ShortcutReference(s)
| Element::InlineImage(s)
| Element::ReferenceImage(s)
| Element::EmptyReferenceImage(s)
| Element::LinkedImage(s)
| Element::FootnoteReference(s)
| Element::Autolink(s)
| Element::HtmlTag(s)
| Element::HtmlEntity(s)
| Element::HugoShortcode(s)
| Element::AttrList(s)
| Element::MystRole(s) => display_len(s, mode),
Element::WikiLink(s) => display_len(s, mode) + 4,
Element::InlineMath(s) => display_len(s, mode) + 2,
Element::DisplayMath(s) => display_len(s, mode) + 4,
Element::EmojiShortcode(s) => display_len(s, mode) + 2,
Element::Strikethrough { content, double } => display_len(content, mode) + if *double { 4 } else { 2 },
Element::Code { content, marker } => display_len(content, mode) + display_len(marker, mode) * 2,
Element::Bold { content, .. } => display_len(content, mode) + 4,
Element::Italic { content, .. } => display_len(content, mode) + 2,
}
}
fn exempt_width(&self, mode: ReflowLengthMode, exemptions: LengthExemptions) -> LineWidth {
let full = self.display_len(mode);
let mut width = LineWidth::plain(full);
match self {
Element::Link(s) | Element::LinkedImage(s) if exemptions.link_urls => {
if let Some(text) = bracketed_text(s, 0) {
width.link_exempt = (2 + display_len(text, mode)).min(full);
}
}
Element::InlineImage(s) if exemptions.link_urls => {
if let Some(alt) = bracketed_text(s, 1) {
width.link_exempt = (3 + display_len(alt, mode)).min(full);
}
}
Element::Code { .. } if exemptions.code_spans => width.code_exempt = 0,
_ => {}
}
width
}
}
fn bracketed_text(s: &str, open: usize) -> Option<&str> {
let bytes = s.as_bytes();
if bytes.get(open) != Some(&b'[') {
return None;
}
let mut depth = 0usize;
let mut in_code_span = false;
let mut escaped = false;
for (i, &byte) in bytes.iter().enumerate().skip(open + 1) {
if escaped {
escaped = false;
continue;
}
match byte {
b'\\' => escaped = true,
b'`' => in_code_span = !in_code_span,
b'[' if !in_code_span => depth += 1,
b']' if !in_code_span => match depth.checked_sub(1) {
Some(next) => depth = next,
None => return s.get(open + 1..i),
},
_ => {}
}
}
None
}
#[derive(Debug, Clone)]
struct EmphasisSpan {
start: usize,
end: usize,
content: String,
is_strong: bool,
is_strikethrough: bool,
uses_underscore: bool,
strikethrough_double: bool,
}
fn extract_emphasis_and_code_spans(text: &str) -> (Vec<EmphasisSpan>, Vec<CodeSpan>) {
let has_emphasis = text.contains(['*', '_', '~']);
let has_code = text.contains('`');
if !has_emphasis && !has_code {
return (Vec::new(), Vec::new());
}
let mut emphasis_spans = Vec::new();
let mut code_spans = Vec::new();
let mut options = Options::empty();
if has_emphasis {
options.insert(Options::ENABLE_STRIKETHROUGH);
}
let mut emphasis_stack: Vec<(usize, bool)> = Vec::new(); let mut strong_stack: Vec<(usize, bool)> = Vec::new();
let mut strikethrough_stack: Vec<usize> = Vec::new();
let parser = Parser::new_ext(text, options).into_offset_iter();
for (event, range) in parser {
match event {
Event::Code(_) => {
code_spans.push(CodeSpan {
start: range.start,
end: range.end,
});
}
Event::Start(Tag::Emphasis) => {
let uses_underscore = text.get(range.start..range.start + 1) == Some("_");
emphasis_stack.push((range.start, uses_underscore));
}
Event::End(TagEnd::Emphasis) => {
if let Some((start_byte, uses_underscore)) = emphasis_stack.pop() {
let content_start = start_byte + 1;
let content_end = range.end - 1;
if content_end > content_start
&& let Some(content) = text.get(content_start..content_end)
{
emphasis_spans.push(EmphasisSpan {
start: start_byte,
end: range.end,
content: content.to_string(),
is_strong: false,
is_strikethrough: false,
uses_underscore,
strikethrough_double: false,
});
}
}
}
Event::Start(Tag::Strong) => {
let uses_underscore = text.get(range.start..range.start + 2) == Some("__");
strong_stack.push((range.start, uses_underscore));
}
Event::End(TagEnd::Strong) => {
if let Some((start_byte, uses_underscore)) = strong_stack.pop() {
let content_start = start_byte + 2;
let content_end = range.end - 2;
if content_end > content_start
&& let Some(content) = text.get(content_start..content_end)
{
emphasis_spans.push(EmphasisSpan {
start: start_byte,
end: range.end,
content: content.to_string(),
is_strong: true,
is_strikethrough: false,
uses_underscore,
strikethrough_double: false,
});
}
}
}
Event::Start(Tag::Strikethrough) => {
strikethrough_stack.push(range.start);
}
Event::End(TagEnd::Strikethrough) => {
if let Some(start_byte) = strikethrough_stack.pop() {
let double = text.get(start_byte..start_byte + 2) == Some("~~");
let marker_len = if double { 2 } else { 1 };
let content_start = start_byte + marker_len;
let content_end = range.end - marker_len;
if content_end > content_start
&& let Some(content) = text.get(content_start..content_end)
{
emphasis_spans.push(EmphasisSpan {
start: start_byte,
end: range.end,
content: content.to_string(),
is_strong: false,
is_strikethrough: true,
uses_underscore: false,
strikethrough_double: double,
});
}
}
}
_ => {}
}
}
emphasis_spans.sort_by_key(|s| s.start);
(emphasis_spans, code_spans)
}
#[derive(Debug, Clone)]
struct CodeSpan {
start: usize,
end: usize,
}
#[derive(Debug, Clone)]
struct LinkSpan {
start: usize,
end: usize,
link_type: Option<LinkType>,
is_image: bool,
is_footnote: bool,
depth: usize,
}
fn extract_link_spans(text: &str, defined_references: Option<&HashSet<String>>) -> Vec<LinkSpan> {
let mut spans = all_link_spans(text, defined_references);
spans.retain(|span| span.depth == 0);
spans
}
fn all_link_spans(text: &str, defined_references: Option<&HashSet<String>>) -> Vec<LinkSpan> {
if !text.contains('[') {
return Vec::new();
}
let mut spans = Vec::new();
let mut options = Options::empty();
options.insert(Options::ENABLE_FOOTNOTES);
let resolve = move |link: BrokenLink<'_>| -> Option<(CowStr<'_>, CowStr<'_>)> {
let atomic = match link.link_type {
LinkType::Shortcut | LinkType::ShortcutUnknown => match defined_references {
Some(defs) => defs.contains(&normalize_reference_label(link.reference.as_ref())),
None => true,
},
_ => true,
};
atomic.then_some((CowStr::Borrowed(""), CowStr::Borrowed("")))
};
let parser = Parser::new_with_broken_link_callback(text, options, Some(resolve)).into_offset_iter();
let mut stack = Vec::new();
for (event, range) in parser {
match event {
Event::Start(Tag::Link { link_type, .. }) => {
stack.push((range.start, Some(link_type), false));
}
Event::Start(Tag::Image { link_type, .. }) => {
stack.push((range.start, Some(link_type), true));
}
Event::End(TagEnd::Link | TagEnd::Image) => {
if let Some((start_byte, link_type, is_image)) = stack.pop() {
spans.push(LinkSpan {
start: start_byte,
end: range.end,
link_type,
is_image,
is_footnote: false,
depth: stack.len(),
});
}
}
Event::FootnoteReference(_) => {
spans.push(LinkSpan {
start: range.start,
end: range.end,
link_type: None,
is_image: false,
is_footnote: true,
depth: stack.len(),
});
}
_ => {}
}
}
spans.sort_by_key(|s| s.start);
spans
}
fn myst_role_len_at(text: &str, absolute_pos: usize, code_spans: &[CodeSpan]) -> Option<usize> {
let bytes = text.as_bytes();
if bytes.first() != Some(&b'{') {
return None;
}
let mut j = 1;
match bytes.get(j) {
Some(&b) if b.is_ascii_alphabetic() || b == b'_' => {}
_ => return None,
}
while let Some(&b) = bytes.get(j) {
if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b':' | b'.') {
j += 1;
} else {
break;
}
}
if bytes.get(j) != Some(&b'}') {
return None;
}
j += 1;
let code_span_start = absolute_pos + j;
if let Ok(idx) = code_spans.binary_search_by_key(&code_span_start, |span| span.start) {
let span = &code_spans[idx];
let code_span_len = span.end - span.start;
return Some(j + code_span_len);
}
None
}
fn inline_math_len_at_start(s: &str) -> Option<usize> {
let bytes = s.as_bytes();
if bytes.first() != Some(&b'$') || bytes.get(1) == Some(&b'$') {
return None;
}
let close = 1 + s[1..].find('$')?;
if bytes.get(close + 1) == Some(&b'$') {
return None;
}
Some(close + 1)
}
#[derive(Clone, Copy, Debug)]
struct PatternMatch {
start: usize,
end: usize,
}
#[derive(Clone, Copy)]
enum PatternCache {
Unsearched,
NotFound,
Found(PatternMatch),
}
impl PatternCache {
fn earliest_in(
&mut self,
remaining: &str,
cursor: usize,
find: impl FnOnce(&str) -> Option<(usize, usize)>,
) -> Option<(usize, usize)> {
let stale = match self {
PatternCache::Found(pm) => pm.start < cursor,
PatternCache::NotFound => false,
PatternCache::Unsearched => true,
};
if stale {
*self = match find(remaining) {
Some((start, end)) => PatternCache::Found(PatternMatch {
start: cursor + start,
end: cursor + end,
}),
None => PatternCache::NotFound,
};
}
match self {
PatternCache::Found(pm) => Some((pm.start - cursor, pm.end - cursor)),
_ => None,
}
}
}
fn parse_markdown_elements_inner(
text: &str,
attr_lists: bool,
myst_roles: bool,
defined_references: Option<&HashSet<String>>,
) -> Vec<Element> {
let mut elements = Vec::new();
let mut remaining = text;
let (emphasis_spans, code_spans) = extract_emphasis_and_code_spans(text);
let link_spans = extract_link_spans(text, defined_references);
let mut cached_wiki_link = PatternCache::Unsearched;
let mut cached_display_math = PatternCache::Unsearched;
let mut cached_inline_math = PatternCache::Unsearched;
let mut cached_emoji = PatternCache::Unsearched;
let mut cached_html_entity = PatternCache::Unsearched;
let mut cached_hugo_shortcode = PatternCache::Unsearched;
let mut cached_html_tag = PatternCache::Unsearched;
let mut cached_next_curly = PatternCache::Unsearched;
let mut link_span_idx = 0usize;
let mut emphasis_span_idx = 0usize;
let mut code_span_idx = 0usize;
while !remaining.is_empty() {
let current_offset = text.len() - remaining.len();
let mut earliest_match: Option<(usize, usize, &str)> = None;
while link_span_idx < link_spans.len() && link_spans[link_span_idx].start < current_offset {
link_span_idx += 1;
}
let next_link: Option<&LinkSpan> = link_spans.get(link_span_idx);
if let Some(span) = next_link {
let pos_in_remaining = span.start - current_offset;
if earliest_match
.as_ref()
.is_none_or(|(start, _, _)| pos_in_remaining < *start)
{
let match_end = span.end - current_offset;
earliest_match = Some((pos_in_remaining, match_end, "link_span"));
}
}
if let Some((start, end)) = cached_wiki_link.earliest_in(remaining, current_offset, |suffix| {
WIKI_LINK_REGEX.find(suffix).map(|m| (m.start(), m.end()))
}) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
{
earliest_match = Some((start, end, "wiki_link"));
}
if let Some((start, end)) = cached_display_math.earliest_in(remaining, current_offset, |suffix| {
DISPLAY_MATH_REGEX.find(suffix).map(|m| (m.start(), m.end()))
}) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
{
earliest_match = Some((start, end, "display_math"));
}
let inline_math_probe = if current_offset > 0 && text.as_bytes()[current_offset - 1] == b'$' {
inline_math_len_at_start(remaining).map(|len| (0, len))
} else {
None
};
if let Some((start, end)) = inline_math_probe.or_else(|| {
cached_inline_math.earliest_in(remaining, current_offset, |suffix| {
INLINE_MATH_REGEX
.find(suffix)
.ok()
.flatten()
.map(|m| (m.start(), m.end()))
})
}) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
{
earliest_match = Some((start, end, "inline_math"));
}
if let Some((start, end)) = cached_emoji.earliest_in(remaining, current_offset, |suffix| {
EMOJI_SHORTCODE_REGEX.find(suffix).map(|m| (m.start(), m.end()))
}) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
{
earliest_match = Some((start, end, "emoji"));
}
if let Some((start, end)) = cached_html_entity.earliest_in(remaining, current_offset, |suffix| {
HTML_ENTITY_REGEX.find(suffix).map(|m| (m.start(), m.end()))
}) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
{
earliest_match = Some((start, end, "html_entity"));
}
if let Some((start, end)) = cached_hugo_shortcode.earliest_in(remaining, current_offset, |suffix| {
HUGO_SHORTCODE_REGEX.find(suffix).map(|m| (m.start(), m.end()))
}) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
{
earliest_match = Some((start, end, "hugo_shortcode"));
}
if let Some((start, end)) = cached_html_tag.earliest_in(remaining, current_offset, |suffix| {
let mut from = 0;
while let Some(m) = HTML_TAG_PATTERN.find(&suffix[from..]) {
let (tag_start, tag_end) = (from + m.start(), from + m.end());
let tag = &suffix[tag_start..tag_end];
let is_url_autolink = tag.starts_with("<http://")
|| tag.starts_with("<https://")
|| tag.starts_with("<mailto:")
|| tag.starts_with("<ftp://")
|| tag.starts_with("<ftps://");
let is_email_autolink = {
let content = tag.trim_start_matches('<').trim_end_matches('>');
EMAIL_PATTERN.is_match(content)
};
if is_url_autolink || is_email_autolink {
from = tag_end;
} else {
return Some((tag_start, tag_end));
}
}
None
}) && earliest_match.as_ref().is_none_or(|(s, _, _)| start < *s)
{
earliest_match = Some((start, end, "html_tag"));
}
let mut next_special = remaining.len();
let mut special_type = "";
let mut pulldown_emphasis: Option<&EmphasisSpan> = None;
let mut attr_list_len: usize = 0;
let mut myst_role_len: usize = 0;
while code_span_idx < code_spans.len() && code_spans[code_span_idx].start < current_offset {
code_span_idx += 1;
}
let next_code_span: Option<&CodeSpan> = code_spans.get(code_span_idx);
if let Some(span) = next_code_span {
let pos_in_remaining = span.start - current_offset;
if pos_in_remaining < next_special {
next_special = pos_in_remaining;
special_type = "pulldown_code";
}
}
let next_curly_pos = cached_next_curly
.earliest_in(remaining, current_offset, |suffix| {
suffix.find('{').map(|pos| (pos, pos + 1))
})
.map(|(start, _)| start);
if myst_roles
&& let Some(pos) = next_curly_pos
&& pos < next_special
&& let Some(role_len) = myst_role_len_at(&remaining[pos..], current_offset + pos, &code_spans)
{
next_special = pos;
special_type = "myst_role";
myst_role_len = role_len;
}
if attr_lists
&& let Some(pos) = next_curly_pos
&& pos < next_special
&& let Some(m) = ATTR_LIST_PATTERN.find(&remaining[pos..])
&& m.start() == 0
{
next_special = pos;
special_type = "attr_list";
attr_list_len = m.end();
}
while emphasis_span_idx < emphasis_spans.len() && emphasis_spans[emphasis_span_idx].start < current_offset {
emphasis_span_idx += 1;
}
if let Some(span) = emphasis_spans.get(emphasis_span_idx) {
let pos_in_remaining = span.start - current_offset;
if pos_in_remaining < next_special {
next_special = pos_in_remaining;
special_type = "pulldown_emphasis";
pulldown_emphasis = Some(span);
}
}
let should_process_markdown_link = if let Some((pos, _, _)) = earliest_match {
pos < next_special
} else {
false
};
if should_process_markdown_link {
let (pos, match_end, pattern_type) = earliest_match.unwrap();
if pos > 0 {
elements.push(Element::Text(remaining[..pos].to_string()));
}
match pattern_type {
"link_span" => {
let span = next_link.unwrap();
let raw_text = remaining[pos..match_end].to_string();
if span.is_footnote {
elements.push(Element::FootnoteReference(raw_text));
} else if span.is_image {
match span.link_type {
Some(LinkType::Inline) => elements.push(Element::InlineImage(raw_text)),
Some(LinkType::Reference)
| Some(LinkType::ReferenceUnknown)
| Some(LinkType::Shortcut)
| Some(LinkType::ShortcutUnknown) => elements.push(Element::ReferenceImage(raw_text)),
Some(LinkType::Collapsed) | Some(LinkType::CollapsedUnknown) => {
elements.push(Element::EmptyReferenceImage(raw_text))
}
_ => elements.push(Element::InlineImage(raw_text)),
}
} else {
match span.link_type {
Some(LinkType::Inline) => {
if raw_text.starts_with('[') && raw_text.contains("![") {
elements.push(Element::LinkedImage(raw_text));
} else {
elements.push(Element::Link(raw_text));
}
}
Some(LinkType::Reference) | Some(LinkType::ReferenceUnknown) => {
elements.push(Element::ReferenceLink(raw_text))
}
Some(LinkType::Collapsed) | Some(LinkType::CollapsedUnknown) => {
elements.push(Element::EmptyReferenceLink(raw_text))
}
Some(LinkType::Shortcut) | Some(LinkType::ShortcutUnknown) => {
elements.push(Element::ShortcutReference(raw_text))
}
Some(LinkType::Autolink) | Some(LinkType::Email) => {
elements.push(Element::Autolink(raw_text))
}
_ => elements.push(Element::Link(raw_text)),
}
}
remaining = &remaining[match_end..];
}
"wiki_link" => {
if let Some(caps) = WIKI_LINK_REGEX.captures(remaining) {
let content = caps.get(1).map_or("", |m| m.as_str());
elements.push(Element::WikiLink(content.to_string()));
remaining = &remaining[match_end..];
} else {
elements.push(Element::Text("[[".to_string()));
remaining = &remaining[2..];
}
}
"display_math" => {
if let Some(caps) = DISPLAY_MATH_REGEX.captures(remaining) {
let math = caps.get(1).map_or("", |m| m.as_str());
elements.push(Element::DisplayMath(math.to_string()));
remaining = &remaining[match_end..];
} else {
elements.push(Element::Text("$$".to_string()));
remaining = &remaining[2..];
}
}
"inline_math" => {
if let Ok(Some(caps)) = INLINE_MATH_REGEX.captures(remaining) {
let math = caps.get(1).map_or("", |m| m.as_str());
elements.push(Element::InlineMath(math.to_string()));
remaining = &remaining[match_end..];
} else {
elements.push(Element::Text("$".to_string()));
remaining = &remaining[1..];
}
}
"emoji" => {
if let Some(caps) = EMOJI_SHORTCODE_REGEX.captures(remaining) {
let emoji = caps.get(1).map_or("", |m| m.as_str());
elements.push(Element::EmojiShortcode(emoji.to_string()));
remaining = &remaining[match_end..];
} else {
elements.push(Element::Text(":".to_string()));
remaining = &remaining[1..];
}
}
"html_entity" => {
elements.push(Element::HtmlEntity(remaining[pos..match_end].to_string()));
remaining = &remaining[match_end..];
}
"hugo_shortcode" => {
elements.push(Element::HugoShortcode(remaining[pos..match_end].to_string()));
remaining = &remaining[match_end..];
}
"html_tag" => {
elements.push(Element::HtmlTag(remaining[pos..match_end].to_string()));
remaining = &remaining[match_end..];
}
_ => unreachable!("unknown pattern type: {}", pattern_type),
}
} else {
if next_special > 0 && next_special < remaining.len() {
elements.push(Element::Text(remaining[..next_special].to_string()));
remaining = &remaining[next_special..];
}
match special_type {
"pulldown_code" => {
let span = next_code_span.unwrap();
let span_len = span.end - span.start;
let code_raw = &remaining[..span_len];
if let Some((content, marker)) = decompose_code_span(code_raw) {
elements.push(Element::Code {
content: content.to_string(),
marker: marker.to_string(),
});
} else {
elements.push(Element::Text(code_raw.to_string()));
}
remaining = &remaining[span_len..];
}
"attr_list" => {
elements.push(Element::AttrList(remaining[..attr_list_len].to_string()));
remaining = &remaining[attr_list_len..];
}
"myst_role" => {
elements.push(Element::MystRole(remaining[..myst_role_len].to_string()));
remaining = &remaining[myst_role_len..];
}
"pulldown_emphasis" => {
let span = pulldown_emphasis.expect("pulldown_emphasis must be set");
let span_len = span.end - span.start;
if span.is_strikethrough {
elements.push(Element::Strikethrough {
content: span.content.clone(),
double: span.strikethrough_double,
});
} else if span.is_strong {
elements.push(Element::Bold {
content: span.content.clone(),
underscore: span.uses_underscore,
});
} else {
elements.push(Element::Italic {
content: span.content.clone(),
underscore: span.uses_underscore,
});
}
remaining = &remaining[span_len..];
}
_ => {
elements.push(Element::Text(remaining.to_string()));
break;
}
}
}
}
let mut merged_elements = Vec::new();
for el in elements {
match el {
Element::Text(s) => {
if let Some(Element::Text(last_s)) = merged_elements.last_mut() {
last_s.push_str(&s);
} else {
merged_elements.push(Element::Text(s));
}
}
other => merged_elements.push(other),
}
}
merged_elements
}
fn source_gap_before(elements: &[Element], idx: usize) -> &str {
let Some(Element::Text(previous)) = idx.checked_sub(1).map(|prev| &elements[prev]) else {
return "";
};
let gap = &previous[previous.trim_end_matches(char::is_whitespace).len()..];
if gap.is_empty() {
""
} else if gap.contains(is_non_breaking_space) {
gap
} else {
" "
}
}
fn push_source_gap(current_line: &mut String, gap: &str) {
if !gap.is_empty() && !current_line.is_empty() && !current_line.ends_with(char::is_whitespace) {
current_line.push_str(gap);
}
}
fn is_setext_or_thematic(text: &str) -> bool {
let mut marker = 0u8;
let mut count = 0usize;
let mut has_space = false;
for &b in text.as_bytes() {
match b {
b' ' | b'\t' => has_space = true,
b'-' | b'=' | b'*' | b'_' => {
if marker == 0 {
marker = b;
} else if b != marker {
return false;
}
count += 1;
}
_ => return false,
}
}
match marker {
b'=' => !has_space,
b'-' => !has_space || count >= 3,
b'*' | b'_' => count >= 3,
_ => false,
}
}
fn starts_block_construct(text: &str) -> bool {
let text = text.trim_start();
let bytes = text.as_bytes();
let Some(&first) = bytes.first() else {
return false;
};
let marker_then_boundary = |len: usize| bytes.len() == len || bytes[len] == b' ' || bytes[len] == b'\t';
match first {
b'>' => true,
b'-' | b'*' | b'+' => marker_then_boundary(1) || is_setext_or_thematic(text),
b'_' | b'=' => is_setext_or_thematic(text),
b':' => is_definition_list_item(text) || text.starts_with(":::"),
b'|' => true,
b'#' => {
let hashes = bytes.iter().take_while(|&&b| b == b'#').count();
hashes <= 6 && marker_then_boundary(hashes)
}
b'`' => bytes.iter().take_while(|&&b| b == b'`').count() >= 3,
b'~' => bytes.iter().take_while(|&&b| b == b'~').count() >= 3,
b'0'..=b'9' => {
let digits = bytes.iter().take_while(|b| b.is_ascii_digit()).count();
digits <= 9
&& text[..digits].trim_start_matches('0') == "1"
&& bytes.len() > digits + 1
&& (bytes[digits] == b'.' || bytes[digits] == b')')
&& (bytes[digits + 1] == b' ' || bytes[digits + 1] == b'\t')
}
b'[' => {
let mut escaped = false;
let mut label_close = None;
for (i, &b) in bytes.iter().enumerate().skip(1) {
if escaped {
escaped = false;
} else if b == b'\\' {
escaped = true;
} else if b == b']' {
label_close = Some(i);
break;
}
}
label_close.is_some_and(|i| bytes.get(i + 1) == Some(&b':'))
}
b'<' => crate::utils::html_block::parse_html_block_start(text).is_some(),
_ => false,
}
}
fn merge_block_construct_continuations(lines: Vec<String>) -> Vec<String> {
let mut merged: Vec<String> = Vec::with_capacity(lines.len());
for line in lines {
merged.push(line);
while merged.len() > 1 && starts_block_construct(merged.last().expect("non-empty")) {
let last = merged.pop().expect("non-empty");
let prev = merged.last_mut().expect("len > 1");
prev.push(' ');
prev.push_str(last.trim_start());
}
}
merged
}
fn reflow_elements_sentence_per_line(elements: &[Element], options: &ReflowOptions) -> Vec<String> {
let abbreviations = get_abbreviations(&options.abbreviations);
let require_sentence_capital = options.require_sentence_capital;
let mut lines = Vec::new();
let mut current_line = String::new();
for (idx, element) in elements.iter().enumerate() {
let is_span = matches!(
element,
Element::Italic { .. } | Element::Bold { .. } | Element::Strikethrough { .. }
);
let piece = match element {
Element::Text(text) => Some(text.clone()),
Element::Italic { content, underscore } => Some(wrap_emphasis(
content,
if *underscore { "_" } else { "*" },
&mut current_line,
source_gap_before(elements, idx),
)),
Element::Bold { content, underscore } => Some(wrap_emphasis(
content,
if *underscore { "__" } else { "**" },
&mut current_line,
source_gap_before(elements, idx),
)),
Element::Strikethrough { content, double } => Some(wrap_emphasis(
content,
if *double { "~~" } else { "~" },
&mut current_line,
source_gap_before(elements, idx),
)),
_ => None,
};
if let Some(piece) = piece {
let appended_span_start = is_span.then_some(current_line.len());
let combined = format!("{current_line}{piece}");
let sentences = split_into_sentences_with_set(
&combined,
&abbreviations,
require_sentence_capital,
appended_span_start,
options.defined_references.as_ref(),
);
let next_bracketed = elements
.get(idx + 1)
.filter(|next| next.opens_with_bracket())
.map(|next| (source_gap_before(elements, idx + 1), next.to_string()));
let closes_before_next = |sentence: &str| -> bool {
let Some((gap, next_str)) = &next_bracketed else {
return true;
};
let mut probe = sentence.to_string();
push_source_gap(&mut probe, gap);
probe.push_str(next_str);
let probe_sentences = split_into_sentences_with_set(
&probe,
&abbreviations,
require_sentence_capital,
None,
options.defined_references.as_ref(),
);
probe_sentences.last().is_some_and(|last| last == next_str)
};
if sentences.len() > 1 {
let mut pending = String::new();
let last = sentences.len() - 1;
for (i, sentence) in sentences.iter().enumerate() {
if !pending.is_empty() {
pending.push(' ');
}
pending.push_str(sentence);
let closed = i < last || (ends_with_sentence_punct(&pending) && closes_before_next(&pending));
if closed && !text_ends_with_abbreviation(&pending, &abbreviations) {
lines.push(std::mem::take(&mut pending));
}
}
current_line = pending;
} else {
let trimmed = combined.trim();
if trimmed.is_empty() {
continue;
}
let ends_with_sentence_punct = ends_with_sentence_punct(trimmed);
if ends_with_sentence_punct
&& !text_ends_with_abbreviation(trimmed, &abbreviations)
&& closes_before_next(trimmed)
{
lines.push(combined.trim_matches(is_breakable_whitespace).to_string());
current_line.clear();
} else {
current_line = combined;
}
}
} else {
let element_str = format!("{element}");
push_source_gap(&mut current_line, source_gap_before(elements, idx));
current_line.push_str(&element_str);
}
}
if !current_line.is_empty() {
lines.push(current_line.trim_matches(is_breakable_whitespace).to_string());
}
lines
}
fn wrap_emphasis(content: &str, marker: &str, current_line: &mut String, gap: &str) -> String {
push_source_gap(current_line, gap);
format!("{marker}{content}{marker}")
}
const BREAK_WORDS: &[&str] = &[
"and",
"or",
"but",
"nor",
"yet",
"so",
"for",
"which",
"that",
"because",
"when",
"if",
"while",
"where",
"although",
"though",
"unless",
"since",
"after",
"before",
"until",
"as",
"once",
"whether",
"however",
"therefore",
"moreover",
"furthermore",
"nevertheless",
"whereas",
];
fn is_clause_punctuation(c: char) -> bool {
matches!(c, ',' | ';' | ':' | '\u{2014}') }
fn clause_break_allowed_after(chars: &[char], i: usize) -> bool {
match chars.get(i + 1) {
None => true,
Some(next) => is_breakable_whitespace(*next),
}
}
fn paren_group_end<'a>(slice: &'a str, element_spans: &[ElementSpan], offset: usize) -> Option<(usize, &'a str)> {
debug_assert!(slice.starts_with('('));
let mut depth: i32 = 0;
for (local_byte, c) in slice.char_indices() {
let global_byte = offset + local_byte;
if depth > 0 && is_inside_element(global_byte, element_spans) {
continue;
}
match c {
'(' => depth += 1,
')' => {
depth -= 1;
if depth == 0 {
let end = local_byte + 1;
let inner = &slice[1..local_byte];
return Some((end, inner));
}
}
_ => {}
}
}
None
}
fn split_at_parenthetical(
text: &str,
line_length: usize,
element_spans: &[ElementSpan],
length_mode: ReflowLengthMode,
) -> Option<(String, String)> {
let min_first_len = ((line_length as f64) * MIN_SPLIT_RATIO) as usize;
if text.starts_with('(')
&& let Some((end_local, inner)) = paren_group_end(text, element_spans, 0)
&& inner.contains(' ')
{
let mut first_end = end_local;
loop {
first_end += text[first_end..]
.char_indices()
.take_while(|(_, c)| !is_breakable_whitespace(*c))
.last()
.map_or(0, |(idx, c)| idx + c.len_utf8());
match element_containing(first_end, element_spans) {
Some(span) => first_end = span.end,
None => break,
}
}
let rest_start = first_end;
let first = &text[..first_end];
if measure(first, 0, element_spans, length_mode).fits(line_length) {
let rest = text[rest_start..].trim_start();
if !rest.is_empty() {
return Some((first.to_string(), rest.to_string()));
}
}
}
let mut best_open_byte: Option<usize> = None;
let mut pos = 0usize;
while pos < text.len() {
if text.as_bytes()[pos] != b'(' {
let c = text[pos..].chars().next().unwrap();
pos += c.len_utf8();
continue;
}
if is_inside_element(pos, element_spans) {
pos += 1;
continue;
}
if let Some((end_local, inner)) = paren_group_end(&text[pos..], element_spans, pos) {
let first = text[..pos].trim_end_matches(is_breakable_whitespace);
let first_len = measure(first, 0, element_spans, length_mode).effective();
if first.len() < pos
&& !first.is_empty()
&& first_len >= min_first_len
&& first_len <= line_length
&& inner.contains(' ')
&& best_open_byte.is_none_or(|prev| pos > prev)
{
best_open_byte = Some(pos);
}
pos += end_local;
} else {
pos += 1;
}
}
let open_byte = best_open_byte?;
let first = text[..open_byte].trim_end_matches(is_breakable_whitespace).to_string();
let rest = text[open_byte..].to_string();
if first.is_empty() || rest.trim().is_empty() {
return None;
}
Some((first, rest))
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct ElementSpan {
start: usize,
end: usize,
full: usize,
link_saving: usize,
code_saving: usize,
is_hard: bool,
}
impl ElementSpan {
fn new(start: usize, len: usize, full: usize, width: LineWidth, is_hard: bool) -> Self {
Self {
start,
end: start + len,
full,
link_saving: full - width.link_exempt,
code_saving: full - width.code_exempt,
is_hard,
}
}
fn contains(&self, pos: usize) -> bool {
pos > self.start && pos < self.end
}
fn within(&self, start: usize, end: usize) -> bool {
self.start >= start && self.end <= end
}
fn exempt_width(&self) -> LineWidth {
LineWidth {
link_exempt: self.full - self.link_saving,
code_exempt: self.full - self.code_saving,
}
}
}
fn compute_element_spans(
elements: &[Element],
mode: ReflowLengthMode,
exemptions: LengthExemptions,
) -> Vec<ElementSpan> {
let mut spans = Vec::new();
let mut offset = 0;
for element in elements {
let len = element.display_len(ReflowLengthMode::Bytes);
if !matches!(element, Element::Text(_)) {
let full = element.display_len(mode);
let width = element.exempt_width(mode, exemptions);
let is_hard = match element {
Element::Bold { content, .. }
| Element::Italic { content, .. }
| Element::Strikethrough { content, .. } => content.contains(['[', '`', '<', '$', '{']),
_ => true,
};
spans.push(ElementSpan::new(offset, len, full, width, is_hard));
}
offset += len;
}
spans
}
fn measure(text: &str, offset: usize, spans: &[ElementSpan], mode: ReflowLengthMode) -> LineWidth {
let full = display_len(text, mode);
let end = offset + text.len();
let mut width = LineWidth::plain(full);
for span in spans.iter().filter(|span| span.within(offset, end)) {
width.link_exempt -= span.link_saving;
width.code_exempt -= span.code_saving;
}
width
}
fn line_width_components(line: &str, options: &ReflowOptions) -> LineWidth {
let raw = display_len(line, options.length_mode);
if !options.length_exemptions.any() {
return LineWidth::plain(raw);
}
let elements = parse_markdown_elements_inner(
line,
options.attr_lists,
options.myst_roles,
options.defined_references.as_ref(),
);
let spans = compute_element_spans(&elements, options.length_mode, options.length_exemptions);
measure(line, 0, &spans, options.length_mode)
}
fn line_width(line: &str, options: &ReflowOptions) -> usize {
line_width_components(line, options).effective()
}
fn line_fits(line: &str, options: &ReflowOptions) -> bool {
display_len(line, options.length_mode) <= options.line_length || line_width(line, options) <= options.line_length
}
fn element_containing(pos: usize, spans: &[ElementSpan]) -> Option<ElementSpan> {
spans.iter().copied().find(|span| span.contains(pos))
}
fn is_inside_element(pos: usize, spans: &[ElementSpan]) -> bool {
element_containing(pos, spans).is_some()
}
const MIN_SPLIT_RATIO: f64 = 0.3;
fn split_at_clause_punctuation(
text: &str,
line_length: usize,
element_spans: &[ElementSpan],
length_mode: ReflowLengthMode,
) -> Option<(String, String)> {
let chars: Vec<char> = text.chars().collect();
let min_first_len = ((line_length as f64) * MIN_SPLIT_RATIO) as usize;
let mut width_acc = LineWidth::default();
let mut search_end_char = 0;
let mut byte = 0usize;
let mut idx = 0usize;
while idx < chars.len() {
let (advance_chars, advance_bytes, width) = match element_spans.iter().find(|s| s.start == byte) {
Some(span) => {
let source = &text[span.start..span.end];
(
source.chars().count(),
source.len(),
measure(source, span.start, element_spans, length_mode),
)
}
None => {
let c = chars[idx];
(
1,
c.len_utf8(),
LineWidth::plain(display_len(&c.to_string(), length_mode)),
)
}
};
if !(width_acc + width).fits(line_length) {
break;
}
width_acc += width;
byte += advance_bytes;
idx += advance_chars;
search_end_char = idx;
}
let mut paren_depth: i32 = 0;
let mut best_pos = None;
for i in (0..search_end_char).rev() {
let byte_start: usize = chars[..i].iter().map(|c| c.len_utf8()).sum();
let byte_after: usize = byte_start + chars[i].len_utf8();
if !is_inside_element(byte_start, element_spans) {
match chars[i] {
')' => paren_depth += 1,
'(' => paren_depth = paren_depth.saturating_sub(1),
_ => {}
}
}
if paren_depth == 0
&& is_clause_punctuation(chars[i])
&& clause_break_allowed_after(&chars, i)
&& !is_inside_element(byte_after, element_spans)
{
best_pos = Some(i);
break;
}
}
let pos = best_pos?;
let first: String = chars[..=pos].iter().collect();
if measure(&first, 0, element_spans, length_mode).effective() < min_first_len {
return None;
}
let rest: String = chars[pos + 1..].iter().collect();
let rest = rest.trim_start().to_string();
if rest.is_empty() {
return None;
}
Some((first, rest))
}
fn paren_depth_map(text: &str, element_spans: &[ElementSpan]) -> Vec<i32> {
let mut map = vec![0i32; text.len()];
let mut depth = 0i32;
for (byte, c) in text.char_indices() {
if !is_inside_element(byte, element_spans) {
match c {
'(' => depth += 1,
')' => depth = depth.saturating_sub(1),
_ => {}
}
}
let end = (byte + c.len_utf8()).min(map.len());
for slot in &mut map[byte..end] {
*slot = depth;
}
}
map
}
fn is_standalone_parenthetical(line: &str) -> bool {
let trimmed = line.trim();
if !trimmed.starts_with('(') {
return false;
}
let Some(close) = trimmed.rfind(')') else {
return false;
};
if trimmed[close + 1..].contains(char::is_whitespace) {
return false;
}
let core = &trimmed[..=close];
let inner = &core[1..core.len() - 1];
if !inner.contains(' ') {
return false;
}
let mut depth = 0i32;
for c in core.chars() {
match c {
'(' => depth += 1,
')' => depth -= 1,
_ => {}
}
if depth < 0 {
return false;
}
}
depth == 0
}
fn split_at_break_word(
text: &str,
line_length: usize,
element_spans: &[ElementSpan],
length_mode: ReflowLengthMode,
) -> Option<(String, String)> {
let lower = text.to_lowercase();
let min_first_len = ((line_length as f64) * MIN_SPLIT_RATIO) as usize;
let mut best_split: Option<(usize, usize)> = None;
let depth_map = paren_depth_map(text, element_spans);
for &word in BREAK_WORDS {
let mut search_start = 0;
while let Some(pos) = lower[search_start..].find(word) {
let abs_pos = search_start + pos;
let preceded_by_space = abs_pos == 0 || text.as_bytes().get(abs_pos - 1) == Some(&b' ');
let followed_by_space = text.as_bytes().get(abs_pos + word.len()) == Some(&b' ');
if preceded_by_space && followed_by_space {
let first_part = text[..abs_pos].trim_end();
let first_part_len = measure(first_part, 0, element_spans, length_mode).effective();
let inside_paren = depth_map.get(abs_pos).is_some_and(|&d| d > 0);
if first_part_len >= min_first_len
&& first_part_len <= line_length
&& !is_inside_element(abs_pos, element_spans)
&& !inside_paren
{
if best_split.is_none_or(|(prev_pos, _)| abs_pos > prev_pos) {
best_split = Some((abs_pos, word.len()));
}
}
}
search_start = abs_pos + word.len();
}
}
let (byte_start, _word_len) = best_split?;
let first = text[..byte_start].trim_end().to_string();
let rest = text[byte_start..].to_string();
if first.is_empty() || rest.trim().is_empty() {
return None;
}
Some((first, rest))
}
fn replaces_whitespace(text: &str, first: &str, rest: &str, element_spans: &[ElementSpan]) -> bool {
if !text.starts_with(first) || !text.ends_with(rest) {
return false;
}
let gap_end = text.len() - rest.len();
gap_end > first.len()
&& text[first.len()..gap_end].chars().all(is_breakable_whitespace)
&& !element_spans
.iter()
.any(|span| first.len() < span.end && span.start < gap_end)
}
fn cascade_split_line(text: &str, options: &ReflowOptions) -> Vec<String> {
let line_length = options.line_length;
let length_mode = options.length_mode;
let attr_lists = options.attr_lists;
let myst_roles = options.myst_roles;
let defined_references = options.defined_references.as_ref();
if line_length == 0 || display_len(text, length_mode) <= line_length {
return vec![text.to_string()];
}
let elements = parse_markdown_elements_inner(text, attr_lists, myst_roles, defined_references);
let element_spans = compute_element_spans(&elements, length_mode, options.length_exemptions);
if measure(text, 0, &element_spans, length_mode).fits(line_length) {
return vec![text.to_string()];
}
let rebased_spans = |start: usize| -> Vec<ElementSpan> {
if start == 0 {
return element_spans.clone();
}
element_spans
.iter()
.filter(|span| span.end > start)
.map(|span| ElementSpan {
start: span.start.saturating_sub(start),
end: span.end.saturating_sub(start),
..*span
})
.collect()
};
let mut result = Vec::new();
let mut start = 0usize;
loop {
let remaining = &text[start..];
let spans = rebased_spans(start);
if measure(remaining, 0, &spans, length_mode).fits(line_length) {
result.push(remaining.to_string());
return result;
}
let at_whitespace = |candidate: Option<(String, String)>| {
candidate.filter(|(first, rest)| replaces_whitespace(remaining, first, rest, &spans))
};
let split = at_whitespace(split_at_parenthetical(remaining, line_length, &spans, length_mode))
.or_else(|| at_whitespace(split_at_clause_punctuation(remaining, line_length, &spans, length_mode)))
.or_else(|| at_whitespace(split_at_break_word(remaining, line_length, &spans, length_mode)));
if let Some((first, rest)) = split {
let consumed = remaining.len().saturating_sub(rest.len());
if consumed == 0 {
break;
}
result.push(first);
start += consumed;
continue;
}
break;
}
let mut fallback_options = options.clone();
fallback_options.break_on_sentences = false;
fallback_options.preserve_breaks = false;
fallback_options.sentence_per_line = false;
fallback_options.semantic_line_breaks = false;
fallback_options.require_sentence_capital = true;
fallback_options.max_list_continuation_indent = None;
fallback_options.defined_references = None;
let remaining = &text[start..];
let tail_elements = if start == 0 {
elements
} else {
parse_markdown_elements_inner(remaining, attr_lists, myst_roles, defined_references)
};
result.extend(reflow_elements(&tail_elements, &fallback_options));
result
}
fn reflow_elements_semantic(elements: &[Element], options: &ReflowOptions) -> Vec<String> {
let sentence_lines = reflow_elements_sentence_per_line(elements, options);
if options.line_length == 0 {
return sentence_lines;
}
let mut result = Vec::new();
for line in sentence_lines {
if line_fits(&line, options) {
result.push(line);
} else {
result.extend(cascade_split_line(&line, options));
}
}
let min_line_len = ((options.line_length as f64) * MIN_SPLIT_RATIO) as usize;
let mut merged: Vec<String> = Vec::with_capacity(result.len());
for line in result {
if !merged.is_empty() && line_width(&line, options) < min_line_len && !line.trim().is_empty() {
if is_standalone_parenthetical(&line) {
merged.push(line);
continue;
}
let prev_ends_at_sentence = {
let trimmed = merged.last().unwrap().trim_end();
trimmed
.chars()
.rev()
.find(|c| !matches!(c, '"' | '\'' | '\u{201D}' | '\u{2019}' | ')' | ']'))
.is_some_and(|c| matches!(c, '.' | '!' | '?'))
};
if !prev_ends_at_sentence {
let prev = merged.last_mut().unwrap();
let combined = format!("{prev} {line}");
if line_fits(&combined, options) {
*prev = combined;
continue;
}
}
}
merged.push(line);
}
merged
}
fn rfind_safe_space(
line: &str,
element_spans: &[ElementSpan],
options: &ReflowOptions,
relax_soft_spans: bool,
) -> Option<usize> {
line.char_indices().rev().map(|(pos, _)| pos).find(|&pos| {
line.as_bytes()[pos] == b' '
&& !is_inside_element_filtered(pos, element_spans, options, relax_soft_spans)
&& !starts_block_construct(&line[pos + 1..])
})
}
fn is_inside_element_filtered(
pos: usize,
spans: &[ElementSpan],
options: &ReflowOptions,
relax_soft_spans: bool,
) -> bool {
spans.iter().any(|span| {
span.contains(pos)
&& (!relax_soft_spans
|| span.is_hard
|| (options.atomic_spans && span.exempt_width().fits(options.line_length)))
})
}
#[derive(Clone, Copy)]
struct Attached<'a> {
text: &'a str,
width: LineWidth,
separator: &'a str,
}
fn break_before_attached(
lines: &mut Vec<String>,
current_line: &mut String,
current_width: &mut LineWidth,
element_spans: &mut Vec<ElementSpan>,
attach: Attached<'_>,
options: &ReflowOptions,
) -> Option<usize> {
let length_mode = options.length_mode;
let last_space = rfind_safe_space(current_line, element_spans, options, false)
.or_else(|| rfind_safe_space(current_line, element_spans, options, true))?;
let before = current_line[..last_space]
.trim_end_matches(is_breakable_whitespace)
.to_string();
let after = current_line[last_space + 1..].to_string();
let after_width = measure(&after, last_space + 1, element_spans, length_mode);
lines.push(before);
let carried = after.len();
let Attached { text, width, separator } = attach;
*current_line = format!("{after}{separator}{text}");
*current_width = after_width + LineWidth::plain(display_len(separator, length_mode)) + width;
rebase_spans_after_break(element_spans, last_space + 1);
Some(carried)
}
fn rebase_spans_after_break(element_spans: &mut Vec<ElementSpan>, carried_start: usize) {
element_spans.retain(|span| span.end > carried_start);
for span in element_spans.iter_mut() {
span.start = span.start.saturating_sub(carried_start);
span.end -= carried_start;
}
}
fn reflow_elements(elements: &[Element], options: &ReflowOptions) -> Vec<String> {
let mut lines = Vec::new();
let mut current_line = String::new();
let mut current_width = LineWidth::default();
let mut current_line_element_spans: Vec<ElementSpan> = Vec::new();
let length_mode = options.length_mode;
let exemptions = options.length_exemptions;
for (idx, element) in elements.iter().enumerate() {
let element_len = element.display_len(length_mode);
let element_width = element.exempt_width(length_mode, exemptions);
let is_hard = match element {
Element::Bold { content, .. }
| Element::Italic { content, .. }
| Element::Strikethrough { content, .. } => content.contains(['[', '`', '<', '$', '{']),
_ => true,
};
let is_adjacent_to_prev = if idx > 0 {
match (&elements[idx - 1], element) {
(Element::Text(t), _) => !t.is_empty() && !t.ends_with(is_breakable_whitespace),
(_, Element::Text(t)) => !t.is_empty() && !t.starts_with(is_breakable_whitespace),
_ => true,
}
} else {
false
};
if let Element::Text(text) = element {
let has_leading_space = text.starts_with(is_breakable_whitespace);
let words: Vec<&str> = split_breakable_words(text).collect();
for (i, word) in words.iter().enumerate() {
let word_width = LineWidth::plain(display_len(word, length_mode));
let is_trailing_punct = word.chars().all(|c| {
matches!(c, ',' | '.' | ':' | ';' | '!' | '?' | ')' | ']' | '}') || is_non_breaking_space(c)
});
let is_first_adjacent = i == 0 && is_adjacent_to_prev;
if is_first_adjacent {
if !(current_width + word_width).fits(options.line_length)
&& !current_width.is_empty()
&& break_before_attached(
&mut lines,
&mut current_line,
&mut current_width,
&mut current_line_element_spans,
Attached {
text: word,
width: word_width,
separator: "",
},
options,
)
.is_some()
{
} else {
current_line.push_str(word);
current_width += word_width;
}
} else if !current_width.is_empty()
&& !(current_width + LineWidth::plain(1) + word_width).fits(options.line_length)
{
if is_trailing_punct {
if break_before_attached(
&mut lines,
&mut current_line,
&mut current_width,
&mut current_line_element_spans,
Attached {
text: word,
width: word_width,
separator: " ",
},
options,
)
.is_none()
{
current_line.push(' ');
current_line.push_str(word);
current_width += LineWidth::plain(1) + word_width;
}
} else if !starts_block_construct(word) {
lines.push(current_line.trim_matches(is_breakable_whitespace).to_string());
current_line = word.to_string();
current_width = word_width;
current_line_element_spans.clear();
} else if break_before_attached(
&mut lines,
&mut current_line,
&mut current_width,
&mut current_line_element_spans,
Attached {
text: word,
width: word_width,
separator: " ",
},
options,
)
.is_some()
{
} else {
if i > 0 || has_leading_space {
current_line.push(' ');
current_width += LineWidth::plain(1);
}
current_line.push_str(word);
current_width += word_width;
}
} else {
let add_space = !current_width.is_empty() && (i > 0 || has_leading_space);
if add_space {
current_line.push(' ');
current_width += LineWidth::plain(1);
}
current_line.push_str(word);
current_width += word_width;
}
}
} else {
let span_info = match element {
Element::Italic { content, underscore } => {
Some((content.as_str(), if *underscore { "_" } else { "*" }, false))
}
Element::Bold { content, underscore } => {
Some((content.as_str(), if *underscore { "__" } else { "**" }, false))
}
Element::Strikethrough { content, double } => {
Some((content.as_str(), if *double { "~~" } else { "~" }, false))
}
Element::Code { content, marker } => Some((content.as_str(), marker.as_str(), true)),
_ => None,
};
let breakable: Option<Vec<&str>> = match span_info {
Some((content, _, is_code)) => {
if is_code {
(!options.atomic_spans && code_span_wraps_losslessly(content))
.then(|| split_breakable_words(content).collect())
} else {
(!options.atomic_spans || element_len > options.line_length)
.then(|| breakable_units(content, options.defined_references.as_ref(), options.attr_lists))
.flatten()
}
}
None => None,
};
if let Some(words) = breakable {
let (_, marker, is_code) = span_info.expect("breakable implies a span");
let n = words.len();
if n == 0 {
let full = format!("{marker}{marker}");
let full_width = LineWidth::plain(display_len(&full, length_mode));
if !is_adjacent_to_prev && !current_width.is_empty() {
current_line.push(' ');
current_width += LineWidth::plain(1);
}
current_line.push_str(&full);
current_width += full_width;
} else {
for (i, word) in words.iter().enumerate() {
let is_first = i == 0;
let is_last = i == n - 1;
let space_start = if is_first && is_code && word.starts_with('`') {
" "
} else {
""
};
let space_end = if is_last && is_code && word.ends_with('`') {
" "
} else {
""
};
let word_str: String = match (is_first, is_last) {
(true, true) => format!("{marker}{space_start}{word}{space_end}{marker}"),
(true, false) => format!("{marker}{space_start}{word}"),
(false, true) => format!("{word}{space_end}{marker}"),
(false, false) => word.to_string(),
};
let word_elements = parse_elements(&word_str, options);
let word_spans = compute_element_spans(&word_elements, length_mode, exemptions);
let word_width = measure(&word_str, 0, &word_spans, length_mode);
let needs_space = if is_first {
!is_adjacent_to_prev && !current_width.is_empty()
} else {
!current_width.is_empty()
};
if needs_space
&& !(current_width + LineWidth::plain(1) + word_width).fits(options.line_length)
&& !starts_block_construct(&word_str)
{
lines.push(current_line.trim_matches(is_breakable_whitespace).to_string());
current_line = word_str;
current_width = word_width;
current_line_element_spans.clear();
for span in word_spans {
current_line_element_spans.push(span);
}
} else {
let mut start_pos = current_line.len();
if needs_space {
current_line.push(' ');
current_width += LineWidth::plain(1);
start_pos += 1;
}
current_line.push_str(&word_str);
current_width += word_width;
for mut span in word_spans {
span.start += start_pos;
span.end += start_pos;
current_line_element_spans.push(span);
}
}
}
}
} else {
let element_str = format!("{element}");
if is_adjacent_to_prev {
if !(current_width + element_width).fits(options.line_length)
&& let Some(carried) = break_before_attached(
&mut lines,
&mut current_line,
&mut current_width,
&mut current_line_element_spans,
Attached {
text: &element_str,
width: element_width,
separator: "",
},
options,
)
{
current_line_element_spans.push(ElementSpan::new(
carried,
element_str.len(),
element_len,
element_width,
is_hard,
));
} else {
let start = current_line.len();
current_line.push_str(&element_str);
current_width += element_width;
current_line_element_spans.push(ElementSpan::new(
start,
element_str.len(),
element_len,
element_width,
is_hard,
));
}
} else if !current_width.is_empty()
&& !(current_width + LineWidth::plain(1) + element_width).fits(options.line_length)
{
if !starts_block_construct(&element_str) {
lines.push(current_line.trim_matches(is_breakable_whitespace).to_string());
current_line.clone_from(&element_str);
current_width = element_width;
current_line_element_spans.clear();
current_line_element_spans.push(ElementSpan::new(
0,
element_str.len(),
element_len,
element_width,
is_hard,
));
} else if let Some(carried) = break_before_attached(
&mut lines,
&mut current_line,
&mut current_width,
&mut current_line_element_spans,
Attached {
text: &element_str,
width: element_width,
separator: " ",
},
options,
) {
let start = carried + 1;
current_line_element_spans.push(ElementSpan::new(
start,
element_str.len(),
element_len,
element_width,
is_hard,
));
} else {
let ends_with_opener =
current_line.ends_with('(') || current_line.ends_with('[') || current_line.ends_with('{');
if !ends_with_opener {
current_line.push(' ');
current_width += LineWidth::plain(1);
}
let start = current_line.len();
current_line.push_str(&element_str);
current_width += element_width;
current_line_element_spans.push(ElementSpan::new(
start,
element_str.len(),
element_len,
element_width,
is_hard,
));
}
} else {
let ends_with_opener =
current_line.ends_with('(') || current_line.ends_with('[') || current_line.ends_with('{');
if !current_width.is_empty() && !ends_with_opener {
current_line.push(' ');
current_width += LineWidth::plain(1);
}
let start = current_line.len();
current_line.push_str(&element_str);
current_width += element_width;
current_line_element_spans.push(ElementSpan::new(
start,
element_str.len(),
element_len,
element_width,
is_hard,
));
}
}
}
}
if !current_line.is_empty() {
lines.push(current_line.trim_end_matches(is_breakable_whitespace).to_string());
}
lines
}
pub fn reflow_markdown(content: &str, options: &ReflowOptions) -> String {
let lines: Vec<&str> = content.lines().collect();
let mut result = Vec::new();
let mut i = 0;
while i < lines.len() {
let line = lines[i];
let trimmed = line.trim();
if trimmed.is_empty() {
result.push(String::new());
i += 1;
continue;
}
if trimmed.starts_with('#') {
result.push(line.to_string());
i += 1;
continue;
}
if trimmed.starts_with(":::") {
result.push(line.to_string());
i += 1;
continue;
}
if trimmed.starts_with("```") || trimmed.starts_with("~~~") {
result.push(line.to_string());
i += 1;
while i < lines.len() {
result.push(lines[i].to_string());
if lines[i].trim().starts_with("```") || lines[i].trim().starts_with("~~~") {
i += 1;
break;
}
i += 1;
}
continue;
}
if calculate_indentation_width_default(line) >= 4 {
result.push(line.to_string());
i += 1;
while i < lines.len() {
let next_line = lines[i];
if calculate_indentation_width_default(next_line) >= 4 || next_line.trim().is_empty() {
result.push(next_line.to_string());
i += 1;
} else {
break;
}
}
continue;
}
if trimmed.starts_with('>') {
let gt_pos = line.find('>').expect("'>' must exist since trimmed.starts_with('>')");
let quote_prefix = line[0..=gt_pos].to_string();
let quote_content = &line[quote_prefix.len()..].trim_start();
let reflowed = reflow_line(quote_content, options);
for reflowed_line in &reflowed {
result.push(format!("{quote_prefix} {reflowed_line}"));
}
i += 1;
continue;
}
if is_horizontal_rule(trimmed) {
result.push(line.to_string());
i += 1;
continue;
}
if is_unordered_list_marker(trimmed) || is_numbered_list_item(trimmed) {
let indent = line.len() - line.trim_start().len();
let indent_str = " ".repeat(indent);
let mut marker_end = indent;
let mut content_start = indent;
if trimmed.chars().next().is_some_and(char::is_numeric) {
if let Some(period_pos) = line[indent..].find('.') {
marker_end = indent + period_pos + 1; content_start = marker_end;
while content_start < line.len() && line.as_bytes().get(content_start) == Some(&b' ') {
content_start += 1;
}
}
} else {
marker_end = indent + 1; content_start = marker_end;
while content_start < line.len() && line.as_bytes().get(content_start) == Some(&b' ') {
content_start += 1;
}
}
let min_continuation_indent = content_start;
let rest = &line[content_start..];
if rest.starts_with("[ ] ") || rest.starts_with("[x] ") || rest.starts_with("[X] ") {
marker_end = content_start + 3; content_start += 4; }
let marker = &line[indent..marker_end];
let mut list_content = vec![trim_preserving_hard_break(&line[content_start..])];
i += 1;
while i < lines.len() {
let next_line = lines[i];
let next_trimmed = next_line.trim();
if is_block_boundary(next_trimmed) {
break;
}
let next_indent = next_line.len() - next_line.trim_start().len();
if next_indent >= min_continuation_indent {
let trimmed_start = next_line.trim_start();
list_content.push(trim_preserving_hard_break(trimmed_start));
i += 1;
} else {
break;
}
}
let combined_content = if options.preserve_breaks {
list_content[0].clone()
} else {
let has_hard_breaks = list_content.iter().any(|line| has_hard_break(line));
if has_hard_breaks {
list_content.join("\n")
} else {
list_content.join(" ")
}
};
let trimmed_marker = marker;
let continuation_spaces = if let Some(max_indent) = options.max_list_continuation_indent {
indent + (content_start - indent).min(max_indent)
} else {
content_start
};
let prefix_length = indent + trimmed_marker.len() + 1;
let adjusted_options = ReflowOptions {
line_length: options.line_length.saturating_sub(prefix_length),
..options.clone()
};
let reflowed = reflow_line(&combined_content, &adjusted_options);
for (j, reflowed_line) in reflowed.iter().enumerate() {
if j == 0 {
result.push(format!("{indent_str}{trimmed_marker} {reflowed_line}"));
} else {
let continuation_indent = " ".repeat(continuation_spaces);
result.push(format!("{continuation_indent}{reflowed_line}"));
}
}
continue;
}
if crate::utils::table_utils::TableUtils::is_potential_table_row(line) {
result.push(line.to_string());
i += 1;
continue;
}
if trimmed.starts_with('[') && line.contains("]:") {
result.push(line.to_string());
i += 1;
continue;
}
if is_definition_list_item(trimmed) {
result.push(line.to_string());
i += 1;
continue;
}
let mut is_single_line_paragraph = true;
if i + 1 < lines.len() {
let next_trimmed = lines[i + 1].trim();
if !is_block_boundary(next_trimmed) {
is_single_line_paragraph = false;
}
}
if is_single_line_paragraph && line_fits(line, options) {
result.push(line.to_string());
i += 1;
continue;
}
let mut paragraph_parts = Vec::new();
let mut current_part = vec![line];
i += 1;
if options.preserve_breaks {
let hard_break_type = if line.strip_suffix('\r').unwrap_or(line).ends_with('\\') {
Some("\\")
} else if line.ends_with(" ") {
Some(" ")
} else {
None
};
let reflowed = reflow_line(line, options);
if let Some(break_marker) = hard_break_type {
if !reflowed.is_empty() {
let mut reflowed_with_break = reflowed;
let last_idx = reflowed_with_break.len() - 1;
if !has_hard_break(&reflowed_with_break[last_idx]) {
reflowed_with_break[last_idx].push_str(break_marker);
}
result.extend(reflowed_with_break);
}
} else {
result.extend(reflowed);
}
} else {
while i < lines.len() {
let prev_line = if !current_part.is_empty() {
current_part.last().unwrap()
} else {
""
};
let next_line = lines[i];
let next_trimmed = next_line.trim();
if is_block_boundary(next_trimmed) {
break;
}
let prev_trimmed = prev_line.trim();
let abbreviations = get_abbreviations(&options.abbreviations);
let ends_with_sentence = (prev_trimmed.ends_with('.')
|| prev_trimmed.ends_with('!')
|| prev_trimmed.ends_with('?')
|| prev_trimmed.ends_with(".*")
|| prev_trimmed.ends_with("!*")
|| prev_trimmed.ends_with("?*")
|| prev_trimmed.ends_with("._")
|| prev_trimmed.ends_with("!_")
|| prev_trimmed.ends_with("?_")
|| prev_trimmed.ends_with(".\"")
|| prev_trimmed.ends_with("!\"")
|| prev_trimmed.ends_with("?\"")
|| prev_trimmed.ends_with(".'")
|| prev_trimmed.ends_with("!'")
|| prev_trimmed.ends_with("?'")
|| prev_trimmed.ends_with(".\u{201D}")
|| prev_trimmed.ends_with("!\u{201D}")
|| prev_trimmed.ends_with("?\u{201D}")
|| prev_trimmed.ends_with(".\u{2019}")
|| prev_trimmed.ends_with("!\u{2019}")
|| prev_trimmed.ends_with("?\u{2019}"))
&& !text_ends_with_abbreviation(
prev_trimmed.trim_end_matches(['*', '_', '"', '\'', '\u{201D}', '\u{2019}']),
&abbreviations,
);
if has_hard_break(prev_line) || (options.sentence_per_line && ends_with_sentence) {
paragraph_parts.push(current_part.join(" "));
current_part = vec![next_line];
} else {
current_part.push(next_line);
}
i += 1;
}
if !current_part.is_empty() {
if current_part.len() == 1 {
paragraph_parts.push(current_part[0].to_string());
} else {
paragraph_parts.push(current_part.join(" "));
}
}
for (j, part) in paragraph_parts.iter().enumerate() {
let reflowed = reflow_line(part, options);
result.extend(reflowed);
if j < paragraph_parts.len() - 1 && !result.is_empty() && !options.sentence_per_line {
let last_idx = result.len() - 1;
if !has_hard_break(&result[last_idx]) {
result[last_idx].push_str(" ");
}
}
}
}
}
let result_text = result.join("\n");
if content.ends_with('\n') && !result_text.ends_with('\n') {
format!("{result_text}\n")
} else {
result_text
}
}
#[derive(Debug, Clone)]
pub struct ParagraphReflow {
pub start_byte: usize,
pub end_byte: usize,
pub reflowed_text: String,
}
#[derive(Debug, Clone)]
pub struct BlockquoteLineData {
pub(crate) content: String,
pub(crate) is_explicit: bool,
pub(crate) prefix: Option<String>,
}
impl BlockquoteLineData {
pub fn explicit(content: String, prefix: String) -> Self {
Self {
content,
is_explicit: true,
prefix: Some(prefix),
}
}
pub fn lazy(content: String) -> Self {
Self {
content,
is_explicit: false,
prefix: None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BlockquoteContinuationStyle {
Explicit,
Lazy,
}
pub fn blockquote_continuation_style(lines: &[BlockquoteLineData]) -> BlockquoteContinuationStyle {
let mut explicit_count = 0usize;
let mut lazy_count = 0usize;
for line in lines.iter().skip(1) {
if line.is_explicit {
explicit_count += 1;
} else {
lazy_count += 1;
}
}
if explicit_count > 0 && lazy_count == 0 {
BlockquoteContinuationStyle::Explicit
} else if lazy_count > 0 && explicit_count == 0 {
BlockquoteContinuationStyle::Lazy
} else if explicit_count >= lazy_count {
BlockquoteContinuationStyle::Explicit
} else {
BlockquoteContinuationStyle::Lazy
}
}
pub fn dominant_blockquote_prefix(lines: &[BlockquoteLineData], fallback: &str) -> String {
let mut counts: std::collections::HashMap<String, (usize, usize)> = std::collections::HashMap::new();
for (idx, line) in lines.iter().enumerate() {
let Some(prefix) = line.prefix.as_ref() else {
continue;
};
counts
.entry(prefix.clone())
.and_modify(|entry| entry.0 += 1)
.or_insert((1, idx));
}
counts
.into_iter()
.max_by(|(_, (count_a, first_idx_a)), (_, (count_b, first_idx_b))| {
count_a.cmp(count_b).then_with(|| first_idx_b.cmp(first_idx_a))
})
.map_or_else(|| fallback.to_string(), |(prefix, _)| prefix)
}
pub(crate) fn should_force_explicit_blockquote_line(content_line: &str) -> bool {
let trimmed = content_line.trim_start();
trimmed.starts_with('>')
|| trimmed.starts_with('#')
|| trimmed.starts_with("```")
|| trimmed.starts_with("~~~")
|| is_unordered_list_marker(trimmed)
|| is_numbered_list_item(trimmed)
|| is_horizontal_rule(trimmed)
|| is_definition_list_item(trimmed)
|| (trimmed.starts_with('[') && trimmed.contains("]:"))
|| trimmed.starts_with(":::")
|| (trimmed.starts_with('<')
&& !trimmed.starts_with("<http")
&& !trimmed.starts_with("<https")
&& !trimmed.starts_with("<mailto:"))
}
pub fn reflow_blockquote_content(
lines: &[BlockquoteLineData],
explicit_prefix: &str,
continuation_style: BlockquoteContinuationStyle,
options: &ReflowOptions,
) -> Vec<String> {
let content_strs: Vec<&str> = lines.iter().map(|l| l.content.as_str()).collect();
let segments = split_into_segments_strs(&content_strs);
let mut reflowed_content_lines: Vec<String> = Vec::new();
for segment in segments {
let hard_break_type = segment.last().and_then(|&line| {
let line = line.strip_suffix('\r').unwrap_or(line);
if line.ends_with('\\') {
Some("\\")
} else if line.ends_with(" ") {
Some(" ")
} else {
None
}
});
let pieces: Vec<&str> = segment
.iter()
.map(|&line| {
if let Some(l) = line.strip_suffix('\\') {
l.trim_end()
} else if let Some(l) = line.strip_suffix(" ") {
l.trim_end()
} else {
line.trim_end()
}
})
.collect();
let segment_text = pieces.join(" ");
let segment_text = segment_text.trim();
if segment_text.is_empty() {
continue;
}
let mut reflowed = reflow_line(segment_text, options);
if let Some(break_marker) = hard_break_type
&& !reflowed.is_empty()
{
let last_idx = reflowed.len() - 1;
if !has_hard_break(&reflowed[last_idx]) {
reflowed[last_idx].push_str(break_marker);
}
}
reflowed_content_lines.extend(reflowed);
}
let mut styled_lines: Vec<String> = Vec::new();
for (idx, line) in reflowed_content_lines.iter().enumerate() {
let force_explicit = idx == 0
|| continuation_style == BlockquoteContinuationStyle::Explicit
|| should_force_explicit_blockquote_line(line);
if force_explicit {
styled_lines.push(format!("{explicit_prefix}{line}"));
} else {
styled_lines.push(line.clone());
}
}
styled_lines
}
fn is_blockquote_content_boundary(content: &str) -> bool {
let trimmed = content.trim();
trimmed.is_empty()
|| is_block_boundary(trimmed)
|| crate::utils::table_utils::TableUtils::is_potential_table_row(content)
|| trimmed.starts_with(":::")
|| crate::utils::is_template_directive_only(content)
|| is_standalone_attr_list(content)
|| is_snippet_block_delimiter(content)
}
fn split_into_segments_strs<'a>(lines: &[&'a str]) -> Vec<Vec<&'a str>> {
let mut segments = Vec::new();
let mut current = Vec::new();
for &line in lines {
current.push(line);
if has_hard_break(line) {
segments.push(current);
current = Vec::new();
}
}
if !current.is_empty() {
segments.push(current);
}
segments
}
fn reflow_blockquote_paragraph_at_line(
content: &str,
lines: &[&str],
target_idx: usize,
options: &ReflowOptions,
) -> Option<ParagraphReflow> {
let mut anchor_idx = target_idx;
let mut target_level = if let Some(parsed) = crate::utils::blockquote::parse_blockquote_prefix(lines[target_idx]) {
parsed.nesting_level
} else {
let mut found = None;
let mut idx = target_idx;
loop {
if lines[idx].trim().is_empty() {
break;
}
if let Some(parsed) = crate::utils::blockquote::parse_blockquote_prefix(lines[idx]) {
found = Some((idx, parsed.nesting_level));
break;
}
if idx == 0 {
break;
}
idx -= 1;
}
let (idx, level) = found?;
anchor_idx = idx;
level
};
let mut para_start = anchor_idx;
while para_start > 0 {
let prev_idx = para_start - 1;
let prev_line = lines[prev_idx];
if prev_line.trim().is_empty() {
break;
}
if let Some(parsed) = crate::utils::blockquote::parse_blockquote_prefix(prev_line) {
if parsed.nesting_level != target_level || is_blockquote_content_boundary(parsed.content) {
break;
}
para_start = prev_idx;
continue;
}
let prev_lazy = prev_line.trim_start();
if is_blockquote_content_boundary(prev_lazy) {
break;
}
para_start = prev_idx;
}
while para_start < lines.len() {
let Some(parsed) = crate::utils::blockquote::parse_blockquote_prefix(lines[para_start]) else {
para_start += 1;
continue;
};
target_level = parsed.nesting_level;
break;
}
if para_start >= lines.len() || para_start > target_idx {
return None;
}
let mut collected: Vec<(usize, BlockquoteLineData)> = Vec::new();
let mut idx = para_start;
while idx < lines.len() {
if !collected.is_empty() && has_hard_break(&collected[collected.len() - 1].1.content) {
break;
}
let line = lines[idx];
if line.trim().is_empty() {
break;
}
if let Some(parsed) = crate::utils::blockquote::parse_blockquote_prefix(line) {
if parsed.nesting_level != target_level || is_blockquote_content_boundary(parsed.content) {
break;
}
collected.push((
idx,
BlockquoteLineData::explicit(trim_preserving_hard_break(parsed.content), parsed.prefix.to_string()),
));
idx += 1;
continue;
}
let lazy_content = line.trim_start();
if is_blockquote_content_boundary(lazy_content) {
break;
}
collected.push((idx, BlockquoteLineData::lazy(trim_preserving_hard_break(lazy_content))));
idx += 1;
}
if collected.is_empty() {
return None;
}
let para_end = collected[collected.len() - 1].0;
if target_idx < para_start || target_idx > para_end {
return None;
}
let line_data: Vec<BlockquoteLineData> = collected.iter().map(|(_, d)| d.clone()).collect();
let fallback_prefix = line_data
.iter()
.find_map(|d| d.prefix.clone())
.unwrap_or_else(|| "> ".to_string());
let explicit_prefix = dominant_blockquote_prefix(&line_data, &fallback_prefix);
let continuation_style = blockquote_continuation_style(&line_data);
let adjusted_line_length = options
.line_length
.saturating_sub(display_len(&explicit_prefix, options.length_mode))
.max(1);
let adjusted_options = ReflowOptions {
line_length: adjusted_line_length,
..options.clone()
};
let styled_lines = reflow_blockquote_content(&line_data, &explicit_prefix, continuation_style, &adjusted_options);
if styled_lines.is_empty() {
return None;
}
let mut start_byte = 0;
for line in lines.iter().take(para_start) {
start_byte += line.len() + 1;
}
let mut end_byte = start_byte;
for line in lines.iter().take(para_end + 1).skip(para_start) {
end_byte += line.len() + 1;
}
let includes_trailing_newline = para_end != lines.len() - 1 || content.ends_with('\n');
if !includes_trailing_newline {
end_byte -= 1;
}
let reflowed_joined = styled_lines.join("\n");
let reflowed_text = if includes_trailing_newline {
if reflowed_joined.ends_with('\n') {
reflowed_joined
} else {
format!("{reflowed_joined}\n")
}
} else if reflowed_joined.ends_with('\n') {
reflowed_joined.trim_end_matches('\n').to_string()
} else {
reflowed_joined
};
Some(ParagraphReflow {
start_byte,
end_byte,
reflowed_text,
})
}
pub fn reflow_paragraph_at_line(content: &str, line_number: usize, line_length: usize) -> Option<ParagraphReflow> {
reflow_paragraph_at_line_with_mode(content, line_number, line_length, ReflowLengthMode::default())
}
pub fn reflow_paragraph_at_line_with_mode(
content: &str,
line_number: usize,
line_length: usize,
length_mode: ReflowLengthMode,
) -> Option<ParagraphReflow> {
let options = ReflowOptions {
line_length,
length_mode,
..Default::default()
};
reflow_paragraph_at_line_with_options(content, line_number, &options)
}
pub fn reflow_paragraph_at_line_with_options(
content: &str,
line_number: usize,
options: &ReflowOptions,
) -> Option<ParagraphReflow> {
if line_number == 0 {
return None;
}
let lines: Vec<&str> = content.lines().collect();
if line_number > lines.len() {
return None;
}
let target_idx = line_number - 1; let target_line = lines[target_idx];
let trimmed = target_line.trim();
if let Some(blockquote_reflow) = reflow_blockquote_paragraph_at_line(content, &lines, target_idx, options) {
return Some(blockquote_reflow);
}
if is_paragraph_boundary(trimmed, target_line) {
return None;
}
let mut para_start = target_idx;
while para_start > 0 {
let prev_idx = para_start - 1;
let prev_line = lines[prev_idx];
let prev_trimmed = prev_line.trim();
if is_paragraph_boundary(prev_trimmed, prev_line) {
break;
}
para_start = prev_idx;
}
let mut para_end = target_idx;
while para_end + 1 < lines.len() {
let next_idx = para_end + 1;
let next_line = lines[next_idx];
let next_trimmed = next_line.trim();
if is_paragraph_boundary(next_trimmed, next_line) {
break;
}
para_end = next_idx;
}
let paragraph_lines = &lines[para_start..=para_end];
let mut start_byte = 0;
for line in lines.iter().take(para_start) {
start_byte += line.len() + 1; }
let mut end_byte = start_byte;
for line in paragraph_lines {
end_byte += line.len() + 1; }
let includes_trailing_newline = para_end != lines.len() - 1 || content.ends_with('\n');
if !includes_trailing_newline {
end_byte -= 1;
}
let paragraph_text = paragraph_lines.join("\n");
let reflowed = reflow_markdown(¶graph_text, options);
let reflowed_text = if includes_trailing_newline {
if reflowed.ends_with('\n') {
reflowed
} else {
format!("{reflowed}\n")
}
} else {
if reflowed.ends_with('\n') {
reflowed.trim_end_matches('\n').to_string()
} else {
reflowed
}
};
Some(ParagraphReflow {
start_byte,
end_byte,
reflowed_text,
})
}
fn decompose_code_span(raw: &str) -> Option<(&str, &str)> {
let marker_len = raw.bytes().take_while(|&b| b == b'`').count();
if marker_len == 0 {
return None;
}
let marker = &raw[..marker_len];
if raw.len() < marker_len * 2 {
return None;
}
let content = &raw[marker_len..raw.len() - marker_len];
Some((content, marker))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn preserves_content_accepts_whitespace_changes_and_rejects_the_rest() {
let accepted: &[(&str, &[&str])] = &[
("one two three", &["one two three"]),
("one two three", &["one two", "three"]),
("one two three", &["one", "two", "three"]),
("one two ", &["one two"]),
("日本語のテキスト", &["日本語の", "テキスト"]),
("_First. Second._", &["_First.", "Second._"]),
];
for (original, reflowed) in accepted {
let reflowed: Vec<String> = reflowed.iter().map(ToString::to_string).collect();
assert!(
preserves_content(original, &reflowed),
"{original:?} -> {reflowed:?} only moves whitespace"
);
}
let rejected: &[(&str, &[&str])] = &[
("one two three", &["one two"]),
("one two", &["one two three"]),
("one two", &["two one"]),
("_First. Second._", &["_First._", "_Second._"]),
("alpha and beta", &["alpha", "andbeta"]),
("mot suivant : autre", &["mot suivant: autre"]),
];
for (original, reflowed) in rejected {
let reflowed: Vec<String> = reflowed.iter().map(ToString::to_string).collect();
assert!(
!preserves_content(original, &reflowed),
"{original:?} -> {reflowed:?} changes the text, not just its line breaks"
);
}
}
#[test]
fn reflow_line_falls_back_to_the_input_when_content_would_change() {
let options = ReflowOptions {
line_length: 40,
..Default::default()
};
let line = "one two three four five six seven eight nine ten";
assert!(preserves_content(line, &reflow_line(line, &options)));
assert_eq!(reflow_line(line, &options), reflow_line_unchecked(line, &options));
}
#[test]
fn cascade_split_line_handles_a_very_long_line_without_overflowing() {
let words: Vec<String> = (0..4000).map(|i| format!("word{i}")).collect();
let line = words.join(" ");
let options = ReflowOptions {
line_length: 80,
length_mode: ReflowLengthMode::Chars,
..Default::default()
};
let out = cascade_split_line(&line, &options);
assert!(out.len() > 1, "a very long line should split into many lines");
for segment in &out {
assert!(
display_len(segment, ReflowLengthMode::Chars) <= 80 || !segment.contains(' '),
"each wrapped line should fit the width (or be a single unbreakable token)"
);
}
let rejoined = out.join(" ");
let original_words: Vec<&str> = line.split(' ').collect();
let result_words: Vec<&str> = rejoined.split_whitespace().collect();
assert_eq!(original_words, result_words, "reflow must preserve all words in order");
}
#[test]
fn test_helper_function_text_ends_with_abbreviation() {
let abbreviations = get_abbreviations(&None);
assert!(text_ends_with_abbreviation("Dr.", &abbreviations));
assert!(text_ends_with_abbreviation("word Dr.", &abbreviations));
assert!(text_ends_with_abbreviation("e.g.", &abbreviations));
assert!(text_ends_with_abbreviation("i.e.", &abbreviations));
assert!(text_ends_with_abbreviation("Mr.", &abbreviations));
assert!(text_ends_with_abbreviation("Mrs.", &abbreviations));
assert!(text_ends_with_abbreviation("Ms.", &abbreviations));
assert!(text_ends_with_abbreviation("Prof.", &abbreviations));
assert!(!text_ends_with_abbreviation("etc.", &abbreviations));
assert!(!text_ends_with_abbreviation("paradigms.", &abbreviations));
assert!(!text_ends_with_abbreviation("programs.", &abbreviations));
assert!(!text_ends_with_abbreviation("items.", &abbreviations));
assert!(!text_ends_with_abbreviation("systems.", &abbreviations));
assert!(!text_ends_with_abbreviation("Dr?", &abbreviations)); assert!(!text_ends_with_abbreviation("Mr!", &abbreviations)); assert!(!text_ends_with_abbreviation("paradigms?", &abbreviations)); assert!(!text_ends_with_abbreviation("word", &abbreviations)); assert!(!text_ends_with_abbreviation("", &abbreviations)); }
#[test]
fn test_footnote_after_period_splits_sentence() {
let text = "First sentence.[^1] Second sentence.";
let sentences = split_into_sentences(text, None);
assert_eq!(
sentences,
vec!["First sentence.[^1]".to_string(), "Second sentence.".to_string()],
"footnote glued to the period should keep the boundary and stay attached to the first sentence"
);
}
#[test]
fn test_multiple_consecutive_footnotes_after_period_splits_sentence() {
let text = "Notes here.[^1][^2] Second sentence.";
let sentences = split_into_sentences(text, None);
assert_eq!(
sentences,
vec!["Notes here.[^1][^2]".to_string(), "Second sentence.".to_string()]
);
}
#[test]
fn test_footnote_before_period_still_splits_sentence() {
let text = "Annotation here[^1]. Second sentence.";
let sentences = split_into_sentences(text, None);
assert_eq!(
sentences,
vec!["Annotation here[^1].".to_string(), "Second sentence.".to_string()]
);
}
#[test]
fn test_mid_sentence_footnote_does_not_split() {
let text = "The system word[^1] more words. Next sentence.";
let sentences = split_into_sentences(text, None);
assert_eq!(
sentences,
vec![
"The system word[^1] more words.".to_string(),
"Next sentence.".to_string()
]
);
}
#[test]
fn test_bare_numeric_bracket_after_period_does_not_split() {
let text = "Citation here.[1] Second sentence.";
let sentences = split_into_sentences(text, None);
assert_eq!(
sentences,
vec![text.to_string()],
"a bare numeric bracket must not be treated as a sentence boundary"
);
}
#[test]
fn test_footnote_glued_to_following_word_does_not_split() {
let text = "First sentence.[^1]Continued glued text.";
let sentences = split_into_sentences(text, None);
assert_eq!(sentences, vec![text.to_string()]);
}
#[test]
fn test_footnote_at_end_of_text_is_preserved() {
let text = "Sentence.[^1]";
let sentences = split_into_sentences(text, None);
assert_eq!(sentences, vec![text.to_string()]);
}
#[test]
fn test_abbreviation_before_footnote_does_not_split() {
let text = "See the notes, e.g.[^1] this one.";
let sentences = split_into_sentences(text, None);
assert_eq!(
sentences,
vec![text.to_string()],
"e.g. is an abbreviation, not a sentence boundary"
);
}
#[test]
fn sentence_boundary_never_falls_inside_an_atomic_construct() {
let cases = [
"Prefix [link. Still link](https://example.com) tail. Next sentence.",
"Prefix [target](<https://example.com/First. Second>) tail. Next sentence.",
"Prefix [text](url \"Title. More\") tail. Next sentence.",
"Prefix  tail. Next sentence.",
"Prefix [ref text. More][ref] tail. Next sentence.",
"Prefix [collapsed. More][] tail. Next sentence.",
"Prefix [[Page name. Title]] tail. Next sentence.",
"Prefix $x. Y$ tail. Next sentence.",
"Prefix $$x. Y$$ tail. Next sentence.",
"Prefix <span title=\"A. B\">x</span> tail. Next sentence.",
"Prefix `code. Still code` tail. Next sentence.",
];
for text in cases {
let sentences = split_into_sentences(text, None);
let (head, tail) = text.rsplit_once(" tail. ").expect("case has a tail");
assert_eq!(
sentences,
vec![format!("{head} tail."), tail.to_string()],
"input {text:?}"
);
}
let text = "Prefix [shortcut. More] tail. Next sentence.";
let whole = vec![
"Prefix [shortcut. More] tail.".to_string(),
"Next sentence.".to_string(),
];
let defined = HashSet::from(["shortcut. more".to_string()]);
assert_eq!(split_into_sentences(text, Some(&defined)), whole);
assert_eq!(split_into_sentences(text, None), whole);
assert_eq!(
split_into_sentences(text, Some(&HashSet::new())),
vec!["Prefix [shortcut.", "More] tail.", "Next sentence."]
);
}
#[test]
fn a_sentence_may_open_with_a_link_or_image() {
for text in [
"Opening sentence. [First. Second](https://example.com)",
"Opening sentence. ",
"Opening sentence. [[First. Second]]",
"Opening sentence. [[first-note|First. Second]]",
"Opening sentence. [Ref link][ref]",
"Opening sentence. [](url) continues.",
"Opening sentence. [][ref] continues.",
"Opening sentence. [![First image][img]](url) continues.",
"Opening sentence. [![First image][]](url) continues.",
"Opening sentence. [![First image][img]][ref] continues.",
] {
let (head, tail) = text.split_once(". ").expect("case has a boundary");
assert_eq!(
split_into_sentences(text, None),
vec![format!("{head}."), tail.to_string()],
"input {text:?}"
);
}
let text = "Opening sentence. [![First image]](url) continues.";
let defined = HashSet::from(["first image".to_string()]);
assert_eq!(
split_into_sentences(text, Some(&defined)),
vec!["Opening sentence.", "[![First image]](url) continues."]
);
assert_eq!(
split_into_sentences(text, Some(&HashSet::new())),
vec![text.to_string()],
"an undefined shortcut is bracketed text, and `!` opens no sentence"
);
assert_eq!(
split_into_sentences("Opening sentence. [](url) continues.", None),
vec](url) continues."]
);
let defined = HashSet::from(["smith 2020".to_string()]);
assert_eq!(
split_into_sentences("Claim ends here. [Smith 2020] more text.", Some(&defined)),
vec!["Claim ends here.", "[Smith 2020] more text."]
);
let none_defined = HashSet::new();
for text in [
"Opening sentence. [first link](https://example.com) continues.",
"Opening sentence. [[first note]] continues.",
"Opening sentence. [[First Note|first note]] continues.",
"Opening sentence. [[Page continues.",
"Opening sentence. [[First] stray]] continues.",
"Opening sentence.  continues.",
"Opening sentence. [1] is the citation.",
"Opening sentence. [First](unterminated",
"Opening sentence. [First][unterminated",
"Opening sentence. [First] (aside) continues.",
"Claim ends here. [Smith 2020]",
"Claim ends here. [Smith 2020] more text.",
"See the RFC. [RFC] More text.",
"Claim ends here. [^Note] more text.",
] {
assert_eq!(
split_into_sentences(text, Some(&none_defined)),
vec![text.to_string()],
"input {text:?}"
);
}
}
#[test]
fn link_opener_is_read_off_the_parse() {
let len = |text: &str, defs: Option<&HashSet<String>>| {
let chars: Vec<char> = text.chars().collect();
let char_offsets = char_byte_offsets(&chars);
let NestedStructure { links, .. } = sentence_structure(text, defs);
let st = SentenceText {
text,
chars: &chars,
char_offsets: &char_offsets,
links: &links,
};
st.link_end_at(0).map_or(0, |end| link_opener_len(&chars, 0, end))
};
let none = HashSet::new();
assert_eq!(len("[text](url)", Some(&none)), 1);
assert_eq!(
len("[text][ref]", Some(&none)),
1,
"a full reference is a link whether or not defined"
);
assert_eq!(len("[text][]", Some(&none)), 1);
assert_eq!(len("", Some(&none)), 2);
assert_eq!(len("[[wiki]]", Some(&none)), 2);
assert_eq!(
len("[[wiki|shown]]", Some(&none)),
7,
"the displayed text starts after the alias pipe"
);
assert_eq!(len("![[img.png|100]]", Some(&none)), 11);
assert_eq!(len("[[wiki|a|b]]", Some(&none)), 7, "the first pipe starts the alias");
assert_eq!(
len("[[wiki|shown]] [[a|b]]", Some(&none)),
7,
"a pipe past the closing `]]` is not this alias"
);
assert_eq!(
len("[a \\] b](url)", Some(&none)),
1,
"an escaped bracket does not close the text"
);
assert_eq!(
len("[](url)", Some(&none)),
1,
"the outer opener is skipped first"
);
for text in [
"[^1]",
"[text](unterminated",
"[text][unterminated",
"[text] (url)",
"[[wiki",
"[[wiki]",
"[[First] stray]]",
"[Smith 2020]",
"[Smith 2020] (see also)",
"[unclosed",
"!bang",
"text",
] {
assert_eq!(len(text, Some(&none)), 0, "input {text:?}");
}
let smith = HashSet::from(["smith 2020".to_string()]);
assert_eq!(len("[Smith 2020]", Some(&smith)), 1);
assert_eq!(len("[Smith 2020]", None), 1);
}
#[test]
fn sentence_per_line_reflow_breaks_before_a_bracket_only_where_the_check_counts() {
let defined = HashSet::from(["spec".to_string()]);
let options = ReflowOptions {
line_length: 120,
sentence_per_line: true,
defined_references: Some(defined.clone()),
..Default::default()
};
for (text, expected) in [
(
"Claim ends here. [Smith](https://example.com) more text. Second sentence.",
vec more text.",
"Second sentence.",
],
),
(
"Wow! [smith](https://example.com) more text. Second sentence.",
vec more text.", "Second sentence."],
),
(
"Claim ends here. [smith](https://example.com) more text. Second sentence.",
vec more text.",
"Second sentence.",
],
),
(
"Claim ends here. [smith][ref] more text. Second sentence.",
vec!["Claim ends here. [smith][ref] more text.", "Second sentence."],
),
(
"Claim ends here.  more text. Second sentence.",
vec more text.", "Second sentence."],
),
(
"Claim ends here.[Link](https://example.com) more text. Second sentence.",
vec more text.",
"Second sentence.",
],
),
(
"See the RFC. [RFC] More text. Second sentence.",
vec!["See the RFC. [RFC] More text.", "Second sentence."],
),
(
"See the spec. [Spec] More text. Second sentence.",
vec!["See the spec.", "[Spec] More text.", "Second sentence."],
),
(
"See the spec. [spec] more text. Second sentence.",
vec!["See the spec. [spec] more text.", "Second sentence."],
),
(
"Claim ends here. [[page|Second sentence]] continues. Third sentence.",
vec![
"Claim ends here.",
"[[page|Second sentence]] continues.",
"Third sentence.",
],
),
(
"Claim ends here. [[Page|second sentence]] continues. Third sentence.",
vec![
"Claim ends here. [[Page|second sentence]] continues.",
"Third sentence.",
],
),
] {
let lines = reflow_line(text, &options);
assert_eq!(lines, expected, "input {text:?}");
assert_eq!(
split_into_sentences(text, Some(&defined)).len(),
expected.len(),
"check count for {text:?}"
);
for line in &lines {
assert_eq!(
split_into_sentences(line, Some(&defined)).len(),
1,
"line {line:?} of {text:?}"
);
}
}
}
#[test]
fn sentence_per_line_reflow_holds_atomic_constructs_whole() {
let options = ReflowOptions {
line_length: 80,
sentence_per_line: true,
..Default::default()
};
let lines = reflow_line(
"Prefix `code. Still code` and [link. Still link](https://example.com) tail. Next sentence.",
&options,
);
assert_eq!(
lines,
vec tail.".to_string(),
"Next sentence.".to_string(),
]
);
let lines = reflow_line(
"Prefix  and [target](<https://example.com/First. Second>) tail. Next sentence.",
&options,
);
assert_eq!(
lines,
vec and [target](<https://example.com/First. Second>) tail.".to_string(),
"Next sentence.".to_string(),
]
);
let lines = reflow_line("First one. Then [link](url) second. Third one.", &options);
assert_eq!(
lines,
vec second.".to_string(),
"Third one.".to_string(),
]
);
}
#[test]
fn test_is_unordered_list_marker() {
assert!(is_unordered_list_marker("- item"));
assert!(is_unordered_list_marker("* item"));
assert!(is_unordered_list_marker("+ item"));
assert!(is_unordered_list_marker("-")); assert!(is_unordered_list_marker("*"));
assert!(is_unordered_list_marker("+"));
assert!(!is_unordered_list_marker("---")); assert!(!is_unordered_list_marker("***")); assert!(!is_unordered_list_marker("- - -")); assert!(!is_unordered_list_marker("* * *")); assert!(!is_unordered_list_marker("*emphasis*")); assert!(!is_unordered_list_marker("-word")); assert!(!is_unordered_list_marker("")); assert!(!is_unordered_list_marker("text")); assert!(!is_unordered_list_marker("# heading")); }
#[test]
fn test_is_block_boundary() {
assert!(is_block_boundary("")); assert!(is_block_boundary("# Heading")); assert!(is_block_boundary("## Level 2")); assert!(is_block_boundary("```rust")); assert!(is_block_boundary("~~~")); assert!(is_block_boundary("> quote")); assert!(is_block_boundary("| cell |")); assert!(is_block_boundary("[link]: http://example.com")); assert!(is_block_boundary("---")); assert!(is_block_boundary("***")); assert!(is_block_boundary("- item")); assert!(is_block_boundary("* item")); assert!(is_block_boundary("+ item")); assert!(is_block_boundary("1. item")); assert!(is_block_boundary("10. item")); assert!(is_block_boundary(": definition")); assert!(is_block_boundary(":::")); assert!(is_block_boundary("::::: {.callout-note}"));
assert!(!is_block_boundary("regular text"));
assert!(!is_block_boundary("*emphasis*")); assert!(!is_block_boundary("[link](url)")); assert!(!is_block_boundary("some words here"));
}
#[test]
fn test_definition_list_boundary_in_single_line_paragraph() {
let options = ReflowOptions {
line_length: 80,
..Default::default()
};
let input = "Term\n: Definition of the term";
let result = reflow_markdown(input, &options);
assert!(
result.contains(": Definition"),
"Definition list item should not be merged into previous line. Got: {result:?}"
);
let lines: Vec<&str> = result.lines().collect();
assert_eq!(lines.len(), 2, "Should remain two separate lines. Got: {lines:?}");
assert_eq!(lines[0], "Term");
assert_eq!(lines[1], ": Definition of the term");
}
#[test]
fn test_is_paragraph_boundary() {
assert!(is_paragraph_boundary("# Heading", "# Heading"));
assert!(is_paragraph_boundary("- item", "- item"));
assert!(is_paragraph_boundary(":::", ":::"));
assert!(is_paragraph_boundary(": definition", ": definition"));
assert!(is_paragraph_boundary("code", " code"));
assert!(is_paragraph_boundary("code", "\tcode"));
assert!(is_paragraph_boundary("| a | b |", "| a | b |"));
assert!(is_paragraph_boundary("a | b", "a | b"));
assert!(!is_paragraph_boundary("regular text", "regular text"));
assert!(!is_paragraph_boundary("text", " text")); }
#[test]
fn test_div_marker_boundary_in_reflow_paragraph_at_line() {
let content = "Some paragraph text here.\n\n::: {.callout-note}\nThis is a callout.\n:::\n";
let result = reflow_paragraph_at_line(content, 3, 80);
assert!(result.is_none(), "Div marker line should not be reflowed");
}
#[test]
fn starts_block_construct_detects_block_openers() {
for case in ["- item", "-", "* item", "*", "+ item", "+", "-\titem"] {
assert!(starts_block_construct(case), "bullet: {case:?}");
}
for case in ["1. item", "1) item", "01. x", "000000001. x", "1.\titem"] {
assert!(starts_block_construct(case), "ordered: {case:?}");
}
for case in ["> quote", ">quote", ">"] {
assert!(starts_block_construct(case), "blockquote: {case:?}");
}
for case in ["# heading", "###### h6", "#", "##"] {
assert!(starts_block_construct(case), "heading: {case:?}");
}
for case in ["```", "```rust", "````", "~~~", "~~~text"] {
assert!(starts_block_construct(case), "fence: {case:?}");
}
for case in ["---", "--", "===", "=", "***", "___", "_ _ _", "- - -"] {
assert!(starts_block_construct(case), "setext/thematic: {case:?}");
}
for case in [
"[^1]: text",
"[^note]:",
"[ref]: http://example.com",
"[wat]: url follows",
] {
assert!(starts_block_construct(case), "definition: {case:?}");
}
for case in ["<div>content", "</div>", "<p>text", "<table>", "<pre>code", "<h1>x"] {
assert!(starts_block_construct(case), "html block: {case:?}");
}
}
#[test]
fn starts_block_construct_allows_ordinary_prose() {
for case in [
"",
"word",
"-5 degrees",
"--flag",
"-item",
"#hashtag",
"####### seven hashes is not a heading",
"1.5 million",
"1234567890. ten digits is not a list marker",
"0000000001. ten digits is not a list marker either",
"2. item",
"7. item",
"0. item",
"42) x",
"123456. item",
"1.",
"1)",
"123456.",
"123456)",
"1.item",
"1:30 pm",
"*emphasis*",
"**bold** text",
"__bold__ text",
"_emphasis_ text",
"`code` span",
"`` double backtick span ``",
"~~strikethrough~~",
"=x",
"== ==",
"(parenthetical)",
"[link](url)",
"[text][ref] more",
"[bracketed] aside",
"[a](b) [ref]: first bracket is a link, not a label",
"[esc\\]: not a close] text",
"<span>inline</span>",
"<b>bold</b>",
"<https://example.com> autolink",
"<mailto:a@b.com>",
"<notarealtag>",
] {
assert!(!starts_block_construct(case), "prose: {case:?}");
}
}
#[test]
fn merge_block_construct_continuations_merges_marker_led_lines() {
let lines = vec![
"First sentence?".to_string(),
"- looks like a list item".to_string(),
"Second sentence.".to_string(),
];
assert_eq!(
merge_block_construct_continuations(lines),
vec![
"First sentence? - looks like a list item".to_string(),
"Second sentence.".to_string(),
]
);
let lines = vec!["- real list content".to_string(), "continuation".to_string()];
assert_eq!(
merge_block_construct_continuations(lines.clone()),
lines,
"first line must never be merged"
);
let lines = vec!["prose".to_string(), "1.".to_string(), "[ref]:".to_string()];
assert_eq!(
merge_block_construct_continuations(lines),
vec!["prose 1. [ref]:".to_string()],
"a merge that creates an opener must fold again"
);
}
#[test]
fn wrap_never_starts_a_line_with_a_block_marker() {
let options = ReflowOptions {
line_length: 25,
..Default::default()
};
let lines = reflow_line(
"Some words here and then - a dash clause that wraps around the limit.",
&options,
);
assert_eq!(
lines,
vec![
"Some words here and",
"then - a dash clause that",
"wraps around the limit."
]
);
for input in [
"Alpha beta gamma delta epsilon - dash clause here to wrap",
"Alpha beta gamma delta epsilon > quote lookalike here to wrap",
"Alpha beta gamma delta epsilon # heading lookalike here to wrap",
"Alpha beta gamma delta epsilon 1. ordered lookalike here to wrap",
"Alpha beta gamma delta epsilon * star clause here to wrap",
"Alpha beta gamma delta epsilon + plus clause here to wrap",
] {
for width in 10..40 {
let options = ReflowOptions {
line_length: width,
..Default::default()
};
for line in reflow_line(input, &options) {
assert!(
!starts_block_construct(&line),
"width {width}: wrapped line opens a block construct: {line:?} (input {input:?})"
);
}
}
}
}
#[test]
fn sentence_per_line_keeps_block_markers_mid_line() {
let options = ReflowOptions {
line_length: 80,
sentence_per_line: true,
..Default::default()
};
let lines = reflow_line(
"Google Calendar (Can't we get rid of this dependency? - I don't really see the need)",
&options,
);
assert_eq!(
lines,
vec!["Google Calendar (Can't we get rid of this dependency? - I don't really see the need)".to_string()]
);
let lines = reflow_line("See section 4? # is the marker we use. Fine.", &options);
assert_eq!(lines, vec!["See section 4? # is the marker we use.", "Fine."]);
let lines = reflow_line("Is this a problem? > I quote someone here.", &options);
assert_eq!(lines, vec!["Is this a problem? > I quote someone here."]);
let lines = reflow_line("Another case! 1. Not a list. More text follows here.", &options);
for line in &lines {
assert!(
!starts_block_construct(line),
"sentence-per-line output opens a block construct: {line:?}"
);
}
}
fn strict_sentence_lines(input: &str, require_sentence_capital: bool) -> Vec<String> {
let options = ReflowOptions {
line_length: 80,
sentence_per_line: true,
require_sentence_capital,
..Default::default()
};
reflow_line(input, &options)
}
#[test]
fn strict_mode_lets_a_sentence_open_with_a_number() {
for (input, expected) in [
(
"The number of items was 5. 2 of them failed.",
vec!["The number of items was 5.", "2 of them failed."],
),
(
"Sometimes we have 2. 3 might be here.",
vec!["Sometimes we have 2.", "3 might be here."],
),
(
"The number of items was 5. 2nd sentence.",
vec!["The number of items was 5.", "2nd sentence."],
),
(
"Released in 2020. 3 of them failed.",
vec!["Released in 2020.", "3 of them failed."],
),
(
"First sentence. 2nd sentence.",
vec!["First sentence.", "2nd sentence."],
),
(
"We met at 6:00 sharp. 6:00 is early.",
vec!["We met at 6:00 sharp.", "6:00 is early."],
),
("Pi is 3.14 roughly. Next.", vec!["Pi is 3.14 roughly.", "Next."]),
(
"A \"Is this a test?\" 2020 was memorable.",
vec!["A \"Is this a test?\"", "2020 was memorable."],
),
] {
assert_eq!(strict_sentence_lines(input, true), expected, "input {input:?}");
}
for input in [
"The count was 5. and that was all.",
"See fig. 3 for details.",
"See no. 5 in the list.",
"See ch. 12 and vol. 3 for more.",
"A \"Is this a test?\" guide to it.",
] {
assert_eq!(
strict_sentence_lines(input, true),
vec![input.to_string()],
"input {input:?}"
);
}
}
#[test]
fn sentence_never_opens_with_an_ordered_list_marker() {
for (input, require_capital, expected) in [
(
"Steps: 1. Do this. 2. Do that.",
true,
vec!["Steps: 1.", "Do this. 2.", "Do that."],
),
(
"First sentence. 1. Do that.",
true,
vec!["First sentence. 1.", "Do that."],
),
("Do this! 2. Do that.", true, vec!["Do this! 2.", "Do that."]),
("Do this. 12) Do that.", true, vec!["Do this. 12) Do that."]),
("Do this. 2. do that.", true, vec!["Do this. 2. do that."]),
("Do this. 2. do that.", false, vec!["Do this. 2.", "do that."]),
(
"Twelve. 1234567890. next one here.",
true,
vec!["Twelve. 1234567890. next one here."],
),
("Do this. 2 more times.", true, vec!["Do this.", "2 more times."]),
("How many? 2.", true, vec!["How many?", "2."]),
("第一句。2. Do that.", true, vec!["第一句。2.", "Do that."]),
("第一句。 2) 第二句。", true, vec!["第一句。 2) 第二句。"]),
("第一句。2 more.", true, vec!["第一句。", "2 more."]),
("第一句。第二句。", true, vec!["第一句。", "第二句。"]),
] {
let lines = strict_sentence_lines(input, require_capital);
assert_eq!(lines, expected, "input {input:?}, require capital {require_capital}");
for line in &lines {
let chars: Vec<char> = line.chars().collect();
assert!(
!opens_ordered_list_marker(&chars),
"line opens with an ordered-list marker: {line:?} (input {input:?})"
);
}
}
}
#[test]
fn opens_ordered_list_marker_matches_the_marker_shape() {
let chars = |s: &str| s.chars().collect::<Vec<char>>();
for text in ["2. x", "1) x", "12. x", "1.\tx", "1234567890. x", "0. x"] {
assert!(opens_ordered_list_marker(&chars(text)), "{text:?} is a marker");
}
for text in ["2.x", "2.", "2)", "2 x", "x. y", "", " 2. x", "2.5 x", "-2. x"] {
assert!(!opens_ordered_list_marker(&chars(text)), "{text:?} is not a marker");
}
}
#[test]
fn inline_math_directly_after_display_math_stays_atomic() {
let options = ReflowOptions {
line_length: 8,
..Default::default()
};
let lines = reflow_line("$$a$$$bb cc dd$ x", &options);
assert_eq!(lines, vec!["$$a$$$bb cc dd$".to_string(), "x".to_string()]);
}
#[test]
fn test_code_span_parsing() {
let elements = parse_markdown_elements_inner("`code`", false, false, None);
assert_eq!(elements.len(), 1);
assert!(matches!(&elements[0], Element::Code { content, marker } if content == "code" && marker == "`"));
let elements = parse_markdown_elements_inner("``code``", false, false, None);
assert_eq!(elements.len(), 1);
assert!(matches!(&elements[0], Element::Code { content, marker } if content == "code" && marker == "``"));
let elements = parse_markdown_elements_inner("``code`inside``", false, false, None);
assert_eq!(elements.len(), 1);
assert!(
matches!(&elements[0], Element::Code { content, marker } if content == "code`inside" && marker == "``")
);
let elements = parse_markdown_elements_inner("`` code ``", false, false, None);
assert_eq!(elements.len(), 1);
assert!(matches!(&elements[0], Element::Code { content, marker } if content == " code " && marker == "``"));
let elements = parse_markdown_elements_inner("`unclosed", false, false, None);
assert_eq!(elements.len(), 1);
assert!(matches!(&elements[0], Element::Text(s) if s == "`unclosed"));
let elements = parse_markdown_elements_inner("`unclosed [link](url)", false, false, None);
assert_eq!(elements.len(), 2);
assert!(matches!(&elements[0], Element::Text(s) if s == "`unclosed "));
assert!(matches!(&elements[1], Element::Link(s) if s == "[link](url)"));
}
#[test]
fn test_reflow_performance_long_input() {
let mut text = String::new();
for i in 1..400 {
let backticks = "`".repeat(i);
text.push_str(&backticks);
text.push(' ');
}
let start = std::time::Instant::now();
let elements = parse_markdown_elements_inner(&text, false, false, None);
let duration = start.elapsed();
assert!(duration.as_millis() < 100, "Parsing took too long: {duration:?}");
assert!(!elements.is_empty());
}
#[test]
fn test_reflow_performance_display_math_heavy() {
let text = "$$a$$".repeat(4000);
let start = std::time::Instant::now();
let elements = parse_markdown_elements_inner(&text, false, false, None);
let duration = start.elapsed();
assert!(duration.as_millis() < 100, "Parsing took too long: {duration:?}");
assert_eq!(elements.len(), 4000);
}
#[test]
fn inline_math_len_at_start_matches_regex_at_slice_start() {
let alphabet = ['$', 'a', ' '];
let mut inputs: Vec<String> = vec![String::new()];
let mut frontier: Vec<String> = vec![String::new()];
for _ in 0..6 {
let mut longer = Vec::new();
for prefix in &frontier {
for ch in alphabet {
let mut s = prefix.clone();
s.push(ch);
longer.push(s);
}
}
inputs.extend(longer.iter().cloned());
frontier = longer;
}
inputs.push("$αβ$x".to_string());
inputs.push("$α$$".to_string());
for s in &inputs {
let expected = INLINE_MATH_REGEX
.find(s)
.ok()
.flatten()
.filter(|m| m.start() == 0)
.map(|m| m.end());
assert_eq!(inline_math_len_at_start(s), expected, "input: {s:?}");
}
}
#[test]
fn inline_math_probe_after_dollar_matches_uncached_parse() {
let cases = [
("$$a$$$b c$ x", r#"[DisplayMath("a"), InlineMath("b c"), Text(" x")]"#),
(
"$$a$$$b$ $$a$$$b$",
r#"[DisplayMath("a"), InlineMath("b"), Text(" "), DisplayMath("a"), InlineMath("b")]"#,
),
(
"$$a$$$ x $y z$",
r#"[DisplayMath("a"), InlineMath(" x "), Text("y z$")]"#,
),
("$$a$$$$ x", r#"[DisplayMath("a"), Text("$$ x")]"#),
("$$a$$$$b$$ x", r#"[DisplayMath("a"), DisplayMath("b"), Text(" x")]"#),
(
"$a$$b$$c$$d$ tail",
r#"[Text("$a"), DisplayMath("b"), Text("c$$d$ tail")]"#,
),
];
for (input, expected) in cases {
let elements = parse_markdown_elements_inner(input, false, false, None);
assert_eq!(format!("{elements:?}"), expected, "input: {input:?}");
}
}
#[test]
fn test_atomic_spans() {
let text_emphasis = "hello **word1 word2**";
let options_disabled = ReflowOptions {
line_length: 18,
atomic_spans: true,
..Default::default()
};
let lines_disabled = reflow_line(text_emphasis, &options_disabled);
assert_eq!(lines_disabled, vec!["hello", "**word1 word2**"]);
let options_enabled = ReflowOptions {
line_length: 18,
atomic_spans: false,
..Default::default()
};
let lines_enabled = reflow_line(text_emphasis, &options_enabled);
assert_eq!(lines_enabled, vec!["hello **word1", "word2**"]);
let text_code = "hello `word1 word2`";
let lines_code_disabled = reflow_line(text_code, &options_disabled);
assert_eq!(lines_code_disabled, vec!["hello", "`word1 word2`"]);
let lines_code_enabled = reflow_line(text_code, &options_enabled);
assert_eq!(lines_code_enabled, vec!["hello `word1", "word2`"]);
let text_code_padding = "hello `` `word1` `word2` ``";
let lines_padding_enabled = reflow_line(text_code_padding, &options_enabled);
assert_eq!(lines_padding_enabled, vec![r#"hello `` `word1`"#, r#"`word2` ``"#]);
let text_attached = "**one two**,";
let options_11 = ReflowOptions {
line_length: 11,
atomic_spans: true,
..Default::default()
};
assert_eq!(reflow_line(text_attached, &options_11), vec!["**one two**,"]);
let options_10 = ReflowOptions {
line_length: 10,
atomic_spans: true,
..Default::default()
};
assert_eq!(reflow_line(text_attached, &options_10), vec!["**one", "two**,"]);
}
#[test]
fn test_emphasis_containing_markers_is_not_split() {
let options = ReflowOptions {
line_length: 5,
atomic_spans: false,
..Default::default()
};
let lines = reflow_line(r#"*foo \*bar*"#, &options);
assert_eq!(lines, vec![r#"*foo \*bar*"#.to_string()]);
}
fn semantic_shape(markdown: &str) -> String {
let mut options = Options::empty();
options.insert(Options::ENABLE_STRIKETHROUGH);
let mut out = String::new();
let push_prose = |out: &mut String, text: &str| {
for c in text.chars() {
if c.is_whitespace() {
if !out.ends_with(char::is_whitespace) {
out.push(' ');
}
} else {
out.push(c);
}
}
};
for event in Parser::new_ext(markdown, options) {
match event {
Event::Text(text) => push_prose(&mut out, &text),
Event::SoftBreak | Event::HardBreak => push_prose(&mut out, " "),
Event::Code(code) => out.push_str(&format!("<code>{code}</code>")),
Event::Start(tag) => out.push_str(&format!("<{tag:?}>")),
Event::End(tag) => out.push_str(&format!("</{tag:?}>")),
other => out.push_str(&format!("{other:?}")),
}
}
out.trim().to_string()
}
#[test]
fn test_wrapping_a_span_never_changes_what_it_parses_to() {
let corpus = [
"_This is a very, very, very, very, very long line with some `code` inside._",
"_alpha beta gamma delta epsilon `a b` zeta eta theta iota kappa lambda_",
"**strong text with `code` and more words than fit on one single line**",
"~~struck text with `code` and more words than fit on one single line~~",
"_emphasis with **nested strong that is quite long** and trailing words_",
"***A doubly nested bold italic span with more words than fit on a line***",
"___Another doubly nested span with more words than fit on a single line___",
"**_mixed strong then emphasis with more words than fit on a single line_**",
"*__mixed emphasis then strong with more words than fit on a single line__*",
"**~~strong strikethrough with more words than fit on a single line here~~**",
"**a * b with a stray marker and plenty more words to pass the budget**",
"_foo `a` bar `b` baz qux quux corge grault garply waldo fred plugh xyzzy_",
"text before _a long emphasis with `code` inside of it here_ and after",
"(_a parenthesized long emphasis with `code` inside of it right here_)",
r#"*foo \*bar baz qux quux corge grault garply waldo fred plugh xyzzy*"#,
"_tab\tseparated `a\tb` words spread out over quite a long emphasis span_",
"_This has [text](<a b c d e f g h i j k>) and trailing words to wrap._",
"_This has [`code`](<a b c d e f g h i j k>) and trailing words to wrap._",
r#"_See [x](https://example.com "a long link title here") and `code` too._"#,
"_A [link with a long label](https://example.com/path) and `code` here._",
"_An image  plus `code` and more text_",
];
for text in corpus {
let expected = semantic_shape(text);
for line_length in [20, 30, 40, 80] {
for atomic_spans in [true, false] {
let options = ReflowOptions {
line_length,
atomic_spans,
..Default::default()
};
let wrapped = reflow_line(text, &options).join("\n");
assert_eq!(
semantic_shape(&wrapped),
expected,
"reflow changed the parse of {text:?} at line_length={line_length} \
atomic_spans={atomic_spans}\n wrapped: {wrapped:?}"
);
}
}
}
}
#[test]
fn test_wrapping_a_span_keeps_whole_the_constructs_pulldown_cannot_see() {
let cases = [
(
"_alpha beta gamma [[a wiki link]] delta epsilon zeta eta_",
"[[a wiki link]]",
),
(
"_alpha beta gamma {{< foo bar >}} delta epsilon zeta eta_",
"{{< foo bar >}}",
),
("_alpha beta gamma $a + b$ delta epsilon zeta eta theta_", "$a + b$"),
("_alpha beta gamma $$a + b$$ delta epsilon zeta eta theta_", "$$a + b$$"),
];
for (text, construct) in cases {
for line_length in [12, 20, 30] {
for atomic_spans in [true, false] {
let options = ReflowOptions {
line_length,
atomic_spans,
..Default::default()
};
let wrapped = reflow_line(text, &options).join("\n");
assert!(
wrapped.contains(construct),
"{construct} was broken at line_length={line_length} \
atomic_spans={atomic_spans}: {wrapped:?}"
);
}
}
}
}
#[test]
fn test_overlong_emphasis_with_nested_code_span_wraps() {
let options = ReflowOptions {
line_length: 80,
atomic_spans: true,
..Default::default()
};
let text = "_This is a very, very, very, very, very, very, very long line that exceeds 80 characters with some `code` inside._";
let lines = reflow_line(text, &options);
assert_eq!(
lines,
vec![
"_This is a very, very, very, very, very, very, very long line that exceeds 80",
"characters with some `code` inside._",
]
);
}
#[test]
fn test_overlong_emphasis_with_nested_strong_wraps() {
let options = ReflowOptions {
line_length: 80,
atomic_spans: true,
..Default::default()
};
let text = "_This is a very, very, very, very, very, very, very long line that exceeds 80 characters with some **bold** inside._";
let lines = reflow_line(text, &options);
assert_eq!(
lines,
vec![
"_This is a very, very, very, very, very, very, very long line that exceeds 80",
"characters with some **bold** inside._",
]
);
}
#[test]
fn test_overlong_doubly_nested_span_wraps() {
let options = ReflowOptions {
line_length: 80,
atomic_spans: true,
..Default::default()
};
let body = "This is a very, very, very, very, very, very, very, very, very, very long line that is emphasised.";
for (open, close) in [
("***", "***"),
("___", "___"),
("**_", "_**"),
("*__", "__*"),
("**~~", "~~**"),
] {
let text = format!("{open}{body}{close}");
assert!(text.len() > options.line_length, "case must start over budget");
let lines = reflow_line(&text, &options);
assert!(
lines.len() > 1,
"{open}...{close} should wrap but stayed on one line: {lines:?}"
);
assert!(
lines.iter().all(|line| line.len() <= options.line_length),
"{open}...{close} left a line over the budget: {lines:?}"
);
assert_eq!(
lines.join(" "),
text,
"{open}...{close} wrapping must only replace a space with a newline"
);
}
}
#[test]
fn test_overlong_span_with_stray_marker_stays_whole() {
let options = ReflowOptions {
line_length: 40,
atomic_spans: true,
..Default::default()
};
let text = "**alpha * beta gamma delta epsilon zeta eta theta iota kappa**";
let lines = reflow_line(text, &options);
assert_eq!(lines, vec![text], "stray marker must keep the span whole");
}
#[test]
fn test_overlong_span_never_breaks_inside_a_nested_reference_link() {
let options = ReflowOptions {
line_length: 30,
atomic_spans: true,
defined_references: Some(HashSet::from([
"ref".to_string(),
"one two three four five six seven".to_string(),
])),
..Default::default()
};
for (text, link) in [
(
"_**alpha [one two three four five six seven][ref] beta gamma delta**_",
"[one two three four five six seven][ref]",
),
(
"**alpha [one two three four five six seven][ref] beta gamma delta**",
"[one two three four five six seven][ref]",
),
(
"_**alpha ![one two three four five six seven][ref] beta gamma delta**_",
"![one two three four five six seven][ref]",
),
(
"_**alpha [one two three four five six seven][] beta gamma delta**_",
"[one two three four five six seven][]",
),
(
"_**alpha [one two three four five six seven] beta gamma delta**_",
"[one two three four five six seven]",
),
] {
let lines = reflow_line(text, &options);
assert!(lines.len() > 1, "over-long span should wrap: {lines:?}");
assert!(
lines.iter().any(|line| line.contains(link)),
"{link} must stay on one line: {lines:?}"
);
assert_eq!(lines.join(" "), text, "wrapping must only move line breaks");
}
}
#[test]
fn test_overlong_span_breaks_inside_an_undefined_shortcut_reference() {
let options = ReflowOptions {
line_length: 30,
atomic_spans: true,
defined_references: Some(HashSet::new()),
..Default::default()
};
let text = "_**alpha [one two three four five six seven] beta gamma delta**_";
let lines = reflow_line(text, &options);
assert!(lines.len() > 1, "over-long span should wrap: {lines:?}");
assert!(
!lines
.iter()
.any(|line| line.contains("[one two three four five six seven]")),
"an undefined shortcut is prose and should break: {lines:?}"
);
assert_eq!(lines.join(" "), text, "wrapping must only move line breaks");
}
#[test]
fn test_overlong_span_never_breaks_inside_a_nested_attr_list() {
let attr = "{.highlight key=\"a b c\"}";
let text = format!("_**alpha beta gamma delta epsilon zeta{attr} eta theta iota kappa**_");
let options = ReflowOptions {
line_length: 20,
atomic_spans: true,
attr_lists: true,
..Default::default()
};
let lines = reflow_line(&text, &options);
assert!(lines.len() > 1, "over-long span should wrap: {lines:?}");
assert!(
lines.iter().any(|line| line.contains(attr)),
"attr list must stay on one line: {lines:?}"
);
assert_eq!(lines.join(" "), text, "wrapping must only move line breaks");
let plain = ReflowOptions {
attr_lists: false,
..options
};
let lines = reflow_line(&text, &plain);
assert!(
!lines.iter().any(|line| line.contains(attr)),
"without the flavor the braces are prose and should break: {lines:?}"
);
assert_eq!(lines.join(" "), text, "wrapping must only move line breaks");
}
#[test]
fn test_overlong_emphasis_never_breaks_inside_nested_code_span() {
let options = ReflowOptions {
line_length: 30,
atomic_spans: true,
..Default::default()
};
let text = "_alpha beta gamma delta epsilon `a b` zeta eta theta iota kappa_";
let lines = reflow_line(text, &options);
assert!(lines.len() > 1, "over-long emphasis should wrap: {lines:?}");
assert!(
lines.iter().any(|line| line.contains("`a b`")),
"nested code span must stay whole with its interior spaces: {lines:?}"
);
for line in &lines {
assert_eq!(
line.matches('`').count() % 2,
0,
"no line may contain half a code span: {line:?}"
);
}
}
#[test]
fn test_definition_list_marker_does_not_start_line() {
let options = ReflowOptions {
line_length: 20,
..Default::default()
};
let lines = reflow_line("This is a term and : definition here.", &options);
for line in &lines {
assert!(
!line.trim_start().starts_with(": "),
"Wrapped line should not start with definition marker: {line}"
);
}
}
#[test]
fn test_div_marker_does_not_start_line() {
let options = ReflowOptions {
line_length: 20,
..Default::default()
};
let lines = reflow_line("This is some text with ::: class marker.", &options);
for line in &lines {
assert!(
!line.trim_start().starts_with(":::"),
"Wrapped line should not start with div marker: {line}"
);
}
}
}