pub fn escape_djot_inline(s: &str) -> String {
escape_djot_inline_in_context("", s, "")
}
pub fn escape_djot_inline_in_context(before: &str, s: &str, after: &str) -> String {
let chars: Vec<char> = s.chars().collect();
let mut result = String::with_capacity(s.len() + s.len() / 8);
for (i, &c) in chars.iter().enumerate() {
if needs_inline_escape(&chars, i, before, after) {
result.push('\\');
}
result.push(c);
}
result
}
fn needs_inline_escape(chars: &[char], i: usize, before: &str, after: &str) -> bool {
let c = chars[i];
match c {
'\\' | '*' | '_' | '`' | '~' | '^' | '[' | ']' | '(' | ')' | '{' | '}' | '|' | '<'
| '\'' | '"' => true,
'-' | '.' => {
let prev = match i {
0 => before.chars().next_back(),
_ => Some(chars[i - 1]),
};
let next = chars.get(i + 1).copied().or_else(|| after.chars().next());
prev == Some(c) || next == Some(c)
}
':' => {
let ahead = chars[i + 1..].iter().copied().chain(after.chars());
let behind = chars[..i].iter().rev().copied().chain(before.chars().rev());
reaches_colon_through_symbol_chars(ahead) || reaches_colon_through_symbol_chars(behind)
}
_ => false,
}
}
fn reaches_colon_through_symbol_chars(mut run: impl Iterator<Item = char>) -> bool {
run.find(|c| !is_symbol_char(*c)) == Some(':')
}
fn is_symbol_char(c: char) -> bool {
c.is_ascii_alphanumeric() || matches!(c, '_' | '+' | '-')
}
pub fn guard_djot_block_start(s: &str) -> String {
let line = s.trim_start_matches(is_line_indent);
let indent = s.len() - line.len();
let (at, insert) = match block_marker_guard(line) {
Some(BlockGuard::EscapeAt(at)) => (indent + at, "\\"),
Some(BlockGuard::EmptyAttributesFirst) => (indent, "{}"),
None => return s.to_string(),
};
let mut out = String::with_capacity(s.len() + insert.len());
out.push_str(&s[..at]);
out.push_str(insert);
out.push_str(&s[at..]);
out
}
enum BlockGuard {
EscapeAt(usize),
EmptyAttributesFirst,
}
fn is_line_indent(c: char) -> bool {
c.is_ascii_whitespace() && c != '\n'
}
fn block_marker_guard(line: &str) -> Option<BlockGuard> {
if let Some(label) = line.strip_prefix('[') {
let colon = 1 + label.find(']')? + 1;
if !line[colon..].starts_with(':') {
return None;
}
return Some(if inside_verbatim(line, colon - 1) {
BlockGuard::EmptyAttributesFirst
} else {
BlockGuard::EscapeAt(colon)
});
}
block_marker_escape_offset(line).map(BlockGuard::EscapeAt)
}
fn inside_verbatim(line: &str, at: usize) -> bool {
let bytes = line.as_bytes();
let mut open: Option<usize> = None;
let mut i = 0;
while i < at {
match bytes[i] {
b'\\' if open.is_none() => {
let escapes = bytes
.get(i + 1)
.is_some_and(|c| c.is_ascii_punctuation() || c.is_ascii_whitespace());
i += if escapes { 2 } else { 1 };
}
b'`' => {
let run = bytes[i..].iter().take_while(|c| **c == b'`').count();
open = match open {
None => Some(run),
Some(fence) if fence == run => None,
still_open => still_open,
};
i += run;
}
_ => i += 1,
}
}
open.is_some()
}
fn block_marker_escape_offset(line: &str) -> Option<usize> {
let first = line.chars().next()?;
let rest = &line[first.len_utf8()..];
match first {
'#' | '>' | '-' | '+' | ':' => Some(0),
'*' => {
if is_thematic_break(line) {
Some(line.find('-').unwrap_or(0))
} else if rest.is_empty() || rest.starts_with([' ', '\t']) {
Some(0)
} else {
None
}
}
'{' => attribute_line_escape_offset(line),
'|' => {
let t = line.trim_end_matches(|c: char| c.is_ascii_whitespace());
(t.len() >= 2 && t.ends_with('|') && !t.ends_with("\\|")).then_some(0)
}
'`' | '~' => {
let fence = line.chars().take_while(|c| *c == first).count();
let spec = line[fence..].trim_matches(|c: char| c.is_ascii_whitespace());
let valid_spec = !spec.contains(|c: char| c.is_ascii_whitespace() || c == '`');
(fence >= 3 && valid_spec).then_some(0)
}
'^' => rest.starts_with(' ').then_some(0),
_ => ordered_list_marker_escape_offset(line),
}
}
fn is_thematic_break(line: &str) -> bool {
let mut marks = 0;
for c in line.chars() {
if matches!(c, '-' | '*') {
marks += 1;
} else if !c.is_ascii_whitespace() {
return false;
}
}
marks >= 3
}
fn ordered_list_marker_escape_offset(line: &str) -> Option<usize> {
fn is_roman_lower(c: u8) -> bool {
matches!(c, b'i' | b'v' | b'x' | b'l' | b'c' | b'd' | b'm')
}
fn is_roman_upper(c: u8) -> bool {
matches!(c, b'I' | b'V' | b'X' | b'L' | b'C' | b'D' | b'M')
}
let bytes = line.as_bytes();
let paren = bytes.first() == Some(&b'(');
let start = usize::from(paren);
let first = *bytes.get(start)?;
let numeral = &bytes[start..];
let numeral_len = if first.is_ascii_digit() {
numeral.iter().take_while(|c| c.is_ascii_digit()).count()
} else if is_roman_lower(first) {
numeral.iter().take_while(|c| is_roman_lower(**c)).count()
} else if is_roman_upper(first) {
numeral.iter().take_while(|c| is_roman_upper(**c)).count()
} else if first.is_ascii_alphabetic() {
1
} else {
return None;
};
let delimiter_at = start + numeral_len;
let delimiter = *bytes.get(delimiter_at)?;
let delimited = if paren {
delimiter == b')'
} else {
matches!(delimiter, b'.' | b')')
};
let ends_marker = bytes
.get(delimiter_at + 1)
.is_none_or(|c| c.is_ascii_whitespace());
(delimited && ends_marker).then_some(if paren { 0 } else { delimiter_at })
}
fn attribute_line_escape_offset(line: &str) -> Option<usize> {
let closed_at = attribute_block_len(line)?;
if !line[closed_at..].chars().all(|c| c.is_ascii_whitespace()) {
return None;
}
Some(attribute_set_escape_offset(line))
}
pub fn guard_djot_attribute_continuation(s: &str) -> String {
if attribute_block_len(s).is_none() {
return s.to_string();
}
let at = attribute_set_escape_offset(s);
let mut out = String::with_capacity(s.len() + 1);
out.push_str(&s[..at]);
out.push('\\');
out.push_str(&s[at..]);
out
}
fn attribute_set_escape_offset(s: &str) -> usize {
let starts_key = s
.as_bytes()
.get(1)
.is_some_and(|c| is_attribute_name_byte(*c));
match s.find('=') {
Some(eq) if starts_key => eq,
_ => 0,
}
}
fn attribute_block_len(line: &str) -> Option<usize> {
#[derive(Clone, Copy)]
enum State {
Start,
Whitespace,
Comment,
ClassFirst,
IdentifierFirst,
Name,
Key,
ValueFirst,
ValueQuoted,
ValueEscape,
}
let mut state = State::Start;
for (i, c) in line.bytes().enumerate() {
state = match (state, c) {
(State::Start, b'{') => State::Whitespace,
(State::Whitespace, b'}') => return Some(i + 1),
(State::Whitespace, b'.') => State::ClassFirst,
(State::Whitespace, b'#') => State::IdentifierFirst,
(State::Whitespace, b'%') => State::Comment,
(State::Whitespace, c) if is_attribute_name_byte(c) => State::Key,
(State::Whitespace, c) if c.is_ascii_whitespace() => State::Whitespace,
(State::Comment, b'%') => State::Whitespace,
(State::Comment, b'}') => return Some(i + 1),
(State::Comment, _) => State::Comment,
(State::ClassFirst | State::IdentifierFirst, c) if is_attribute_name_byte(c) => {
State::Name
}
(State::Name, c) if is_attribute_name_byte(c) => State::Name,
(State::Name, c) if c.is_ascii_whitespace() => State::Whitespace,
(State::Name, b'}') => return Some(i + 1),
(State::Key, c) if is_attribute_name_byte(c) => State::Key,
(State::Key, b'=') => State::ValueFirst,
(State::ValueFirst, c) if is_attribute_name_byte(c) => State::Name,
(State::ValueFirst, b'"') => State::ValueQuoted,
(State::ValueQuoted, b'"') => State::Whitespace,
(State::ValueQuoted, b'\\') => State::ValueEscape,
(State::ValueQuoted | State::ValueEscape, _) => State::ValueQuoted,
_ => return None,
};
}
None
}
fn is_attribute_name_byte(c: u8) -> bool {
c.is_ascii_alphanumeric() || matches!(c, b':' | b'_' | b'-')
}
const IMAGE_SOURCE_ESCAPES: [(char, &str); 14] = [
('\n', "0A"),
('\r', "0D"),
('(', "28"),
(')', "29"),
('*', "2A"),
('<', "3C"),
('>', "3E"),
('^', "5E"),
('_', "5F"),
('`', "60"),
('{', "7B"),
('|', "7C"),
('}', "7D"),
('~', "7E"),
];
fn image_source_escape(rest: &str) -> Option<char> {
let mut after = rest;
let mut percent = false;
loop {
let hex = after.get(..2)?;
let decoded = match IMAGE_SOURCE_ESCAPES.iter().find(|(_, code)| *code == hex) {
Some((c, _)) => *c,
None if hex == "5C" && after.len() == 2 => '\\',
None if hex == "25" => {
percent = true;
after = &after[2..];
continue;
}
None => return None,
};
return Some(if percent { '%' } else { decoded });
}
}
pub fn reads_as_image_source_escape(rest: &str) -> bool {
image_source_escape(rest).is_some()
}
pub fn decode_djot_image_source(src: &str) -> String {
let mut out = String::with_capacity(src.len());
let mut rest = src;
while let Some(at) = rest.find('%') {
out.push_str(&rest[..at]);
rest = &rest[at + 1..];
match image_source_escape(rest) {
Some(c) => {
out.push(c);
rest = &rest[2..];
}
None => out.push('%'),
}
}
out.push_str(rest);
out
}
pub fn keep_djot_edge_whitespace(line: &str) -> String {
let opens = line.starts_with(|c: char| c.is_ascii_whitespace());
let ends = line.ends_with(|c: char| c.is_ascii_whitespace());
if !opens && !ends {
return line.to_string();
}
let mut out = String::with_capacity(line.len() + 4);
if opens {
out.push_str("{}");
}
out.push_str(line);
if ends {
out.push_str("{}");
}
out
}
pub fn plain_text_to_djot(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut first = true;
for line in s.split('\n') {
if line.is_empty() {
continue;
}
if !first {
out.push_str("\n\n");
}
first = false;
let guarded = guard_djot_block_start(&escape_djot_inline(line));
out.push_str(&keep_djot_edge_whitespace(&guarded));
}
out
}
pub fn needs_djot_escaping(s: &str) -> bool {
plain_text_to_djot(s) != s
}
pub fn djot_round_trip_is_lossy(s: &str) -> bool {
use crate::parser_tools::djot_options::DjotImportOptions;
crate::parser_tools::content_parser::djot_to_plain_text(
&plain_text_to_djot(s),
&DjotImportOptions::default(),
) != s
}
#[cfg(test)]
mod tests {
use super::super::content_parser::djot_to_plain_text;
use super::*;
use crate::parser_tools::djot_options::DjotImportOptions;
#[test]
fn escaped_plain_text_parses_back_to_itself() {
for original in [
"plain prose, nothing special",
"a *starred* word",
"snake_case and more_snake_case",
"code `backticks` here",
"brackets [like this] and (parens)",
"a title: The Lighthouse [Revised]",
"# not a heading",
"- not a list item",
"1. not an ordered list",
"12) also not an ordered list",
"> not a quote",
"+ not a list",
": not a definition",
"a backslash \\ alone",
"tilde ~sub~ and caret ^sup^",
"braces {attr} and a pipe | here",
"an angle <bracket>",
"line one\nline two",
"- leading marker\nand a second line",
"1. first\n2. second\n3. third",
"unicode — em dash, ellipsis …, quotes “ ”",
"We met at 10:30:45 sharp.",
"Use std::vector and a::b.",
"a :smile: b",
"::: warning",
"He said \"hi\" and 'bye'.",
"Pages 10--20, then---nothing... or ----",
"I. The beginning",
"mix. then stir",
"A.\tIntroduction",
"A.",
"(c) third",
"B) plan",
] {
let djot = plain_text_to_djot(original);
let round_tripped = djot_to_plain_text(&djot, &DjotImportOptions::default());
assert_eq!(
round_tripped, *original,
"escaping {original:?} produced {djot:?}, which parsed back as \
{round_tripped:?} — the escape is not round-trip safe"
);
}
}
#[test]
fn ordinary_prose_is_left_untouched() {
for plain in [
"Just an ordinary remark.",
"Two sentences. Both ordinary!",
"A question? Yes.",
"",
] {
assert_eq!(plain_text_to_djot(plain), plain);
assert!(!needs_djot_escaping(plain), "{plain:?} needs no escaping");
}
}
#[test]
fn text_with_markup_characters_is_reported_as_needing_escaping() {
for plain in ["a *star*", "# heading-ish", "1. listish", "under_score"] {
assert!(needs_djot_escaping(plain), "{plain:?} must need escaping");
}
}
#[test]
fn a_blank_line_collapses_because_no_djot_can_produce_one() {
let round_tripped =
djot_to_plain_text(&plain_text_to_djot("a\n\nb"), &DjotImportOptions::default());
assert_eq!(round_tripped, "a\nb");
}
#[test]
fn lossiness_is_not_detectable_by_string_comparison_alone() {
assert!(
!needs_djot_escaping("a\n\nb"),
"the escape happens to reproduce the input byte-for-byte here"
);
assert!(
djot_round_trip_is_lossy("a\n\nb"),
"…but the round trip still loses the blank line, and a migration must be able \
to see that"
);
}
#[test]
fn edge_whitespace_survives_the_round_trip() {
for original in [
"trailing spaces are content ",
"\tShe opened the door.",
" two leading spaces",
" \t mixed \t ",
" ",
"\t",
"\u{c}form feed\u{c}",
"\rcarriage return\r",
"\t- a marker behind a tab",
"first\n\tsecond, indented\nthird ",
"no edge whitespace",
] {
let djot = plain_text_to_djot(original);
let round_tripped = djot_to_plain_text(&djot, &DjotImportOptions::default());
assert_eq!(
round_tripped, original,
"{original:?} was written as {djot:?}"
);
assert!(!djot_round_trip_is_lossy(original), "{original:?}");
}
assert_eq!(plain_text_to_djot("\tx"), "{}\tx");
assert_eq!(plain_text_to_djot("x "), "x {}");
assert_eq!(plain_text_to_djot(" "), "{} {}");
assert_eq!(plain_text_to_djot("no edges"), "no edges");
}
#[test]
fn an_image_source_is_decoded_only_where_the_writer_encodes() {
for (source, name) in [
("a%28b%29.png", "a(b).png"),
("a%5C", "a\\"),
("a\\%5C", "a\\\\"),
("%2528.png", "%28.png"),
("%255C", "%5C"),
("%252528", "%2528"),
("%%2529", "%%29"),
(
"https://example.com/100%25.png",
"https://example.com/100%25.png",
),
("a%5Cb.png", "a%5Cb.png"),
("%5C%5C.png", "%5C%5C.png"),
("%25", "%25"),
("%2525", "%2525"),
("a%20b%2F", "a%20b%2F"),
("a%28b", "a(b"),
("%2", "%2"),
("%", "%"),
("é%28é", "é(é"),
("%é", "%é"),
] {
assert_eq!(decode_djot_image_source(source), name, "{source:?}");
}
assert!(reads_as_image_source_escape("28"));
assert!(reads_as_image_source_escape("5C"));
assert!(reads_as_image_source_escape("2528"));
assert!(reads_as_image_source_escape("25255C"));
assert!(!reads_as_image_source_escape("5Cb"));
assert!(!reads_as_image_source_escape("25"));
assert!(!reads_as_image_source_escape("25.png"));
assert!(!reads_as_image_source_escape("2F"));
}
#[test]
fn a_multi_paragraph_comment_body_round_trips() {
let body = "First paragraph of the note.\nA second one, with *emphasis* typed literally.";
let djot = plain_text_to_djot(body);
assert_eq!(
djot_to_plain_text(&djot, &DjotImportOptions::default()),
body
);
}
#[test]
fn an_ordered_list_guard_escapes_the_delimiter_not_the_digit() {
assert_eq!(guard_djot_block_start("1. text"), "1\\. text");
assert_eq!(guard_djot_block_start("42) text"), "42\\) text");
}
#[test]
fn smart_punctuation_and_symbols_are_escaped_only_where_the_parser_would_act() {
for (plain, escaped) in [
("10:30:45", "10\\:30\\:45"),
("std::vector", "std\\:\\:vector"),
(":+1:", "\\:+1\\:"),
("It's \"so\"", "It\\'s \\\"so\\\""),
("a--b---c", "a\\-\\-b\\-\\-\\-c"),
("Wait...", "Wait\\.\\.\\."),
("a..b", "a\\.\\.b"),
("At 10:30.", "At 10:30."),
(
"https://example.com:8080/path",
"https://example.com:8080/path",
),
("Il dit : oui", "Il dit : oui"),
("well-known", "well-known"),
("End. Start.", "End. Start."),
(":-) and :)", ":-\\) and :\\)"),
] {
assert_eq!(escape_djot_inline(plain), escaped, "escaping {plain:?}");
}
}
#[test]
fn a_piece_is_escaped_against_the_text_around_it() {
assert_eq!(escape_djot_inline_in_context("a-", "-b", ""), "\\-b");
assert_eq!(escape_djot_inline_in_context("", "a-", "-b"), "a\\-");
assert_eq!(escape_djot_inline_in_context("Wait.", ".", ". what"), "\\.");
assert_eq!(escape_djot_inline_in_context("", "10:3", "0:45"), "10\\:3");
assert_eq!(escape_djot_inline_in_context("10:3", "0:45", ""), "0\\:45");
assert_eq!(
escape_djot_inline_in_context("std:", ":vector", ""),
"\\:vector"
);
assert_eq!(escape_djot_inline_in_context("a ", "-b", ""), "-b");
assert_eq!(escape_djot_inline_in_context("", "10:3", " 45"), "10:3");
}
#[test]
fn a_letter_or_roman_numeral_list_marker_is_guarded() {
for (line, guarded) in [
("A. Capital", "A\\. Capital"),
("I. The beginning", "I\\. The beginning"),
("iv.\tFourth", "iv\\.\tFourth"),
("mix. then stir", "mix\\. then stir"),
("A.", "A\\."),
("B) plan", "B\\) plan"),
("(c) third", "\\(c) third"),
("(12) twelve", "\\(12) twelve"),
("Mr. Smith", "Mr. Smith"),
("3.14 is pi", "3.14 is pi"),
("Did. Done.", "Did. Done."),
("Mild. Very mild.", "Mild. Very mild."),
("ab. c", "ab. c"),
("(c)opyright", "(c)opyright"),
] {
assert_eq!(guard_djot_block_start(line), guarded, "guarding {line:?}");
}
}
#[test]
fn a_marker_behind_leading_whitespace_is_guarded() {
assert_eq!(guard_djot_block_start("\t- item"), "\t\\- item");
assert_eq!(guard_djot_block_start(" # title"), " \\# title");
assert_eq!(guard_djot_block_start(" \t1.\tstep"), " \t1\\.\tstep");
assert_eq!(guard_djot_block_start(" I. roman"), " I\\. roman");
assert_eq!(guard_djot_block_start(" plain"), " plain");
}
#[test]
fn markup_that_opens_a_block_is_guarded_without_breaking_the_markup() {
assert_eq!(guard_djot_block_start("*-*"), "*\\-*");
assert_eq!(guard_djot_block_start("*- - -*"), "*\\- - -*");
assert_eq!(guard_djot_block_start("*bold* text"), "*bold* text");
assert_eq!(guard_djot_block_start("[^1]: text"), "[^1]\\: text");
assert_eq!(guard_djot_block_start("[a\\]: b](u)"), "[a\\]\\: b](u)");
assert_eq!(guard_djot_block_start("[text](u): more"), "[text](u): more");
assert_eq!(guard_djot_block_start("[`]: x`](u)"), "{}[`]: x`](u)");
assert_eq!(
guard_djot_block_start(" [``a`]: x``](u)"),
" {}[``a`]: x``](u)"
);
assert_eq!(guard_djot_block_start("[\\`]: x"), "[\\`]\\: x");
assert_eq!(guard_djot_block_start("[`a`]: x"), "[`a`]\\: x");
assert_eq!(guard_djot_block_start("{-debug=true-}"), "{-debug\\=true-}");
assert_eq!(guard_djot_block_start("{-a=b c=d-} "), "{-a\\=b c=d-} ");
assert_eq!(guard_djot_block_start("{-gone-}"), "{-gone-}");
assert_eq!(
guard_djot_block_start("{-a=b-} and more"),
"{-a=b-} and more"
);
assert_eq!(guard_djot_block_start("{+a=b+}"), "{+a=b+}");
}
#[test]
fn a_run_that_would_continue_an_attribute_set_is_guarded() {
for (piece, guarded) in [
("{-x=5-}", "{-x\\=5-}"),
("{-=C-}", "{-\\=C-}"),
("{-a=b c=d-} then", "{-a\\=b c=d-} then"),
("{-x:y=z-}", "{-x:y\\=z-}"),
("{-gone-}", "{-gone-}"),
("{-a b-}", "{-a b-}"),
("{+a=b+}", "{+a=b+}"),
("*{-a=b-}*", "*{-a=b-}*"),
("plain", "plain"),
("", ""),
] {
assert_eq!(
guard_djot_attribute_continuation(piece),
guarded,
"guarding {piece:?}"
);
}
for struck in ["x=5", "=C", "a=b c=d", "debug=true", "gone"] {
let piece = guard_djot_attribute_continuation(&format!("{{-{struck}-}}"));
let line = format!("See{{width=6 height=9}}{piece}");
let read = djot_to_plain_text(&line, &DjotImportOptions::default());
assert!(read.ends_with(struck), "{line:?} read back as {read:?}");
}
}
#[test]
fn the_other_block_markers_are_guarded() {
for (line, guarded) in [
("* bullet", "\\* bullet"),
("*", "\\*"),
("* * *", "\\* * *"),
("| a | b |", "\\| a | b |"),
("```rust", "\\```rust"),
("~~~", "\\~~~"),
("^ caption", "\\^ caption"),
("{.class}", "\\{.class}"),
(":::", "\\:::"),
("| a", "| a"),
("``a`` b", "``a`` b"),
] {
assert_eq!(guard_djot_block_start(line), guarded, "guarding {line:?}");
}
}
}