use std::{borrow::Cow, ops::Range};
const PROTECTION_BYTES: &[u8] = &[
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x0b, 0x0c, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15,
0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x7f,
];
pub(crate) struct ProtectedInlineTags<'a> {
pub source: Cow<'a, str>,
pub brackets: Option<(u8, u8)>,
}
pub(crate) fn protect_inline_tag_pairs<'a>(
source: &'a str,
excluded: &[Range<usize>],
) -> ProtectedInlineTags<'a> {
let mut available = PROTECTION_BYTES
.iter()
.copied()
.filter(|byte| !source.as_bytes().contains(byte));
let (Some(open), Some(close)) = (available.next(), available.next()) else {
return ProtectedInlineTags {
source: Cow::Borrowed(source),
brackets: None,
};
};
struct TagSpan {
start: usize,
length: usize,
name: String,
closing: bool,
}
let mut offset = 0;
let mut tags = Vec::new();
for token in tokenize_inline(source, true) {
let length = match token {
InlineToken::Text(text) => text.len(),
InlineToken::Hashtag(tag) => tag.len() + 1,
InlineToken::Embed(target) => target.len() + 5,
InlineToken::WikiLink(target) => target.len() + 4,
InlineToken::Tag {
raw, name, closing, ..
} => {
let length = raw.len();
if !is_backslash_escaped(source, offset)
&& !excluded.iter().any(|range| range.contains(&offset))
{
tags.push(TagSpan {
start: offset,
length,
name,
closing,
});
}
length
}
};
offset += length;
}
debug_assert_eq!(offset, source.len());
let mut paired = vec![false; tags.len()];
let mut open_tags = Vec::new();
for (index, tag) in tags.iter().enumerate() {
if !tag.closing {
open_tags.push(index);
continue;
}
let matching = if tag.name.is_empty() {
open_tags.pop()
} else {
open_tags
.iter()
.rposition(|open| tags[*open].name == tag.name)
.map(|position| open_tags.remove(position))
};
if let Some(open) = matching {
paired[open] = true;
paired[index] = true;
}
}
let mut bytes = source.as_bytes().to_vec();
let mut changed = false;
for (tag, paired) in tags.iter().zip(paired) {
if paired {
bytes[tag.start] = open;
bytes[tag.start + tag.length - 1] = close;
changed = true;
}
}
ProtectedInlineTags {
source: changed
.then(|| String::from_utf8(bytes).expect("ASCII sentinels preserve UTF-8"))
.map_or(Cow::Borrowed(source), Cow::Owned),
brackets: changed.then_some((open, close)),
}
}
pub(crate) fn restore_protected(value: &str, brackets: Option<(u8, u8)>) -> String {
let Some((open, close)) = brackets else {
return value.to_string();
};
value
.chars()
.map(|character| match character {
character if character == char::from(open) => '[',
character if character == char::from(close) => ']',
character => character,
})
.collect()
}
fn is_backslash_escaped(source: &str, offset: usize) -> bool {
source.as_bytes()[..offset]
.iter()
.rev()
.take_while(|byte| **byte == b'\\')
.count()
% 2
== 1
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum InlineToken<'a> {
Text(&'a str),
Hashtag(&'a str),
Embed(&'a str),
WikiLink(&'a str),
Tag {
raw: &'a str,
name: String,
value: Option<String>,
closing: bool,
},
}
pub fn tokenize_inline(input: &str, start_is_hashtag_boundary: bool) -> Vec<InlineToken<'_>> {
let mut tokens = Vec::new();
let mut cursor = 0;
let mut text_start = 0;
while cursor < input.len() {
let ch = input[cursor..]
.chars()
.next()
.expect("cursor is on a character boundary");
if ch == '#' {
if let Some(end) = hashtag_end(input, cursor, start_is_hashtag_boundary) {
push_text(&mut tokens, input, text_start, cursor);
tokens.push(InlineToken::Hashtag(&input[cursor + 1..end]));
cursor = end;
text_start = cursor;
continue;
}
} else if ch == '!' && input[cursor..].starts_with("![[") {
if let Some(end) = wikilink_end(input, cursor + 1) {
push_text(&mut tokens, input, text_start, cursor);
tokens.push(InlineToken::Embed(&input[cursor + 3..end]));
cursor = end + 2;
text_start = cursor;
continue;
}
} else if ch == '[' && input[cursor..].starts_with("[[") {
if let Some(end) = wikilink_end(input, cursor) {
push_text(&mut tokens, input, text_start, cursor);
tokens.push(InlineToken::WikiLink(&input[cursor + 2..end]));
cursor = end + 2;
text_start = cursor;
continue;
}
}
if ch != '[' {
cursor += ch.len_utf8();
continue;
}
let Some(end) = find_tag_end(input, cursor + 1) else {
cursor += 1;
continue;
};
let body = input[cursor + 1..end].trim();
let Some((name, value, closing)) = parse_tag(body) else {
cursor = end + 1;
continue;
};
push_text(&mut tokens, input, text_start, cursor);
tokens.push(InlineToken::Tag {
raw: &input[cursor..=end],
name,
value,
closing,
});
cursor = end + 1;
text_start = cursor;
}
if text_start < input.len() {
tokens.push(InlineToken::Text(&input[text_start..]));
}
tokens
}
fn push_text<'a>(tokens: &mut Vec<InlineToken<'a>>, input: &'a str, start: usize, end: usize) {
if start < end {
tokens.push(InlineToken::Text(&input[start..end]));
}
}
fn hashtag_end(input: &str, start: usize, start_is_boundary: bool) -> Option<usize> {
let has_boundary = if start == 0 {
start_is_boundary
} else {
input[..start]
.chars()
.next_back()
.is_some_and(char::is_whitespace)
};
if !has_boundary {
return None;
}
let mut end = start + 1;
let first = input.get(end..)?.chars().next()?;
if !(first.is_alphanumeric() || first == '_') {
return None;
}
end += first.len_utf8();
for character in input[end..].chars() {
if character.is_alphanumeric() || matches!(character, '_' | '-' | '/') {
end += character.len_utf8();
} else {
break;
}
}
Some(end)
}
fn wikilink_end(input: &str, start: usize) -> Option<usize> {
let body_start = start + 2;
let relative_end = input.get(body_start..)?.find("]]")?;
let end = body_start + relative_end;
let body = &input[body_start..end];
(!body.is_empty() && body.trim() == body && !body.contains(['\n', '\r', '['])).then_some(end)
}
fn parse_tag(body: &str) -> Option<(String, Option<String>, bool)> {
if body.is_empty() {
return None;
}
let (closing, body) = body
.strip_prefix('/')
.map_or((false, body), |rest| (true, rest.trim()));
let (name, value) = body.split_once('=').map_or((body, None), |(name, value)| {
(name.trim(), parse_value(value.trim()))
});
let name = name.trim().to_ascii_lowercase();
if (!closing && name.is_empty())
|| !name
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
|| (closing && value.is_some())
{
return None;
}
Some((name, value, closing))
}
fn parse_value(value: &str) -> Option<String> {
if !value.starts_with('"') {
return Some(value.to_string());
}
let mut output = String::new();
let mut escaped = false;
let mut chars = value[1..].chars();
while let Some(ch) = chars.next() {
if escaped {
output.push(ch);
escaped = false;
} else if ch == '\\' {
escaped = true;
} else if ch == '"' {
return chars.as_str().trim().is_empty().then_some(output);
} else {
output.push(ch);
}
}
None
}
fn find_tag_end(input: &str, start: usize) -> Option<usize> {
let mut quoted = false;
let mut escaped = false;
for (offset, ch) in input.get(start..)?.char_indices() {
if escaped {
escaped = false;
} else if quoted && ch == '\\' {
escaped = true;
} else if ch == '"' {
quoted = !quoted;
} else if ch == ']' && !quoted {
return Some(start + offset);
}
}
None
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tokenizes_styles_and_quoted_links() {
let tokens = tokenize_inline("a[red]b[/red][link=\"A]B\"]c[/link]", true);
assert_eq!(tokens.len(), 7);
assert!(matches!(
&tokens[4],
InlineToken::Tag {
name,
value: Some(value),
..
} if name == "link" && value == "A]B"
));
}
#[test]
fn tokenizes_hashtags_and_wikilinks() {
assert_eq!(
tokenize_inline("See #rust/lang and [[Rust Notes]].", true),
vec![
InlineToken::Text("See "),
InlineToken::Hashtag("rust/lang"),
InlineToken::Text(" and "),
InlineToken::WikiLink("Rust Notes"),
InlineToken::Text("."),
]
);
}
#[test]
fn distinguishes_embeds_from_wikilinks() {
assert_eq!(
tokenize_inline("![[assets/image.png]] and [[Image note]]", true),
vec![
InlineToken::Embed("assets/image.png"),
InlineToken::Text(" and "),
InlineToken::WikiLink("Image note"),
]
);
}
#[test]
fn invalid_embeds_stay_literal() {
for source in ["![[]]", "![[ image.png]]", "![[image.png ]]", "![[a[b]]"] {
assert_eq!(
tokenize_inline(source, true),
vec![InlineToken::Text(source)]
);
}
}
#[test]
fn hashtags_require_a_whitespace_boundary_and_name() {
assert_eq!(
tokenize_inline("title#fragment # #中文 #开发-日志", true),
vec![
InlineToken::Text("title#fragment # "),
InlineToken::Hashtag("中文"),
InlineToken::Text(" "),
InlineToken::Hashtag("开发-日志"),
]
);
}
#[test]
fn hashtags_require_line_start_or_whitespace_before_hash() {
assert_eq!(
tokenize_inline("[#2652 (#paren foo:#suffix #valid\n#next\t#tab", true),
vec![
InlineToken::Text("[#2652 (#paren foo:#suffix "),
InlineToken::Hashtag("valid"),
InlineToken::Text("\n"),
InlineToken::Hashtag("next"),
InlineToken::Text("\t"),
InlineToken::Hashtag("tab"),
]
);
assert_eq!(
tokenize_inline("#fragment", false),
vec![InlineToken::Text("#fragment")]
);
}
#[test]
fn invalid_wikilinks_remain_text() {
assert_eq!(
tokenize_inline("[[]] [[ spaced ]] [[open", true),
vec![InlineToken::Text("[[]] [[ spaced ]] [[open")]
);
}
}