use std::collections::{HashMap, HashSet};
use crate::js::js_trim;
use crate::read::RX;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum TokenType {
Directive,
Wiki,
Embed,
Footnote,
Math,
Param,
}
impl TokenType {
pub fn as_str(self) -> &'static str {
match self {
TokenType::Directive => "directive",
TokenType::Wiki => "wiki",
TokenType::Embed => "embed",
TokenType::Footnote => "footnote",
TokenType::Math => "math",
TokenType::Param => "param",
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct InlineToken {
pub token_type: TokenType,
pub start: usize,
pub end: usize,
pub kind: String,
pub label: String,
pub label_start: usize,
pub label_end: usize,
pub attributes: Option<String>,
pub target: String,
pub piped: bool,
pub formula: String,
pub name: String,
}
impl InlineToken {
fn new(token_type: TokenType, start: usize, end: usize) -> InlineToken {
InlineToken {
token_type,
start,
end,
kind: String::new(),
label: String::new(),
label_start: start,
label_end: start,
attributes: None,
target: String::new(),
piped: false,
formula: String::new(),
name: String::new(),
}
}
}
const MAX_LABEL_DEPTH: usize = 32;
pub fn inline_tokens(source: &str) -> Vec<InlineToken> {
let blocks = Blocks::read(source);
let mut out = Vec::new();
for chunk in &blocks.chunks {
let (v, map) = chunk.text(source);
let closers = label_closers(&v);
let mut s = Scanner { v: &v, map: &map, source, defs: &blocks.defs, out: &mut out, cell: chunk.cell, closers, math_fails_until: 0 };
s.scan(0, v.len(), 0);
}
out
}
pub fn split_wiki(inner: &str) -> (String, String, bool) {
let b = inner.as_bytes();
let mut i = 0;
while i < b.len() {
if b[i] == b'\\' {
i += 2;
continue;
}
if b[i] == b'|' {
return (unescape_wiki(&inner[..i]), unescape_wiki(&inner[i + 1..]), true);
}
i += 1;
}
(unescape_wiki(inner), String::new(), false)
}
fn unescape_wiki(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut chars = s.chars().peekable();
while let Some(c) = chars.next() {
if c == '\\'
&& let Some(&n) = chars.peek()
&& matches!(n, '\\' | '|' | ']' | '[')
{
out.push(n);
chars.next();
continue;
}
out.push(c);
}
js_trim(&out).to_string()
}
struct Chunk {
spans: Vec<(usize, usize)>,
cell: bool,
}
impl Chunk {
fn text(&self, source: &str) -> (String, Vec<usize>) {
let mut v = String::new();
let mut map = Vec::new();
for (k, &(a, b)) in self.spans.iter().enumerate() {
if k > 0 {
let prev_end = self.spans[k - 1].1;
v.push('\n');
map.push(prev_end);
}
v.push_str(&source[a..b]);
map.extend(a..b);
}
map.push(self.spans.last().map_or(0, |s| s.1));
(v, map)
}
}
#[derive(Clone, Copy, PartialEq, Debug)]
enum Container {
Quote,
Item(usize),
}
#[derive(PartialEq, Debug)]
enum Leaf {
None,
Paragraph,
Fenced(u8, usize),
Indented,
Html(u8),
Table,
}
struct Blocks {
chunks: Vec<Chunk>,
defs: HashSet<String>,
}
fn indentation(line: &str, p: usize) -> (usize, usize) {
let b = line.as_bytes();
let (mut col, mut i) = (0, p);
while i < b.len() {
match b[i] {
b' ' => col += 1,
b'\t' => col += 4 - col % 4,
_ => break,
}
i += 1;
}
(col, i)
}
fn consume_columns(line: &str, p: usize, n: usize) -> usize {
let b = line.as_bytes();
let (mut col, mut i) = (0, p);
while i < b.len() && col < n {
match b[i] {
b' ' => col += 1,
b'\t' => col += 4 - col % 4,
_ => break,
}
i += 1;
}
i
}
fn is_blank(s: &str) -> bool {
s.bytes().all(|c| c == b' ' || c == b'\t')
}
fn list_marker(line: &str, p: usize) -> Option<(usize, bool, bool, bool, usize)> {
let (ind, q) = indentation(line, p);
if ind > 3 {
return None;
}
let b = line.as_bytes();
let (marker_end, ordered, one) = match b.get(q) {
Some(b'-' | b'*' | b'+') => (q + 1, false, true),
Some(c) if c.is_ascii_digit() => {
let mut k = q;
while k < b.len() && b[k].is_ascii_digit() && k - q < 10 {
k += 1;
}
if k - q > 9 || !matches!(b.get(k), Some(b'.' | b')')) {
return None;
}
(k + 1, true, &line[q..k] == "1")
}
_ => return None,
};
if marker_end < b.len() && b[marker_end] != b' ' && b[marker_end] != b'\t' {
return None;
}
let rest = &line[marker_end..];
let width = ind + (marker_end - q);
if is_blank(rest) {
return Some((width + 1, ordered, one, true, line.len()));
}
let (spaces, content) = indentation(line, marker_end);
if spaces >= 5 {
return Some((width + 1, ordered, one, false, marker_end + 1));
}
Some((width + spaces.max(1), ordered, one, false, content))
}
fn fence_open(line: &str, p: usize) -> Option<(u8, usize)> {
let (ind, q) = indentation(line, p);
if ind > 3 {
return None;
}
let b = line.as_bytes();
let ch = *b.get(q)?;
if ch != b'`' && ch != b'~' {
return None;
}
let mut k = q;
while k < b.len() && b[k] == ch {
k += 1;
}
let len = k - q;
if len < 3 || ch == b'`' && line[k..].contains('`') {
return None;
}
Some((ch, len))
}
fn fence_close(line: &str, p: usize, ch: u8, len: usize) -> bool {
let (ind, q) = indentation(line, p);
if ind > 3 {
return false;
}
let b = line.as_bytes();
let mut k = q;
while k < b.len() && b[k] == ch {
k += 1;
}
k - q >= len && is_blank(&line[k..])
}
fn thematic(rest: &str) -> bool {
let (ind, q) = indentation(rest, 0);
if ind > 3 {
return false;
}
let b = rest.as_bytes();
let Some(&ch) = b.get(q).filter(|c| matches!(c, b'-' | b'*' | b'_')) else { return false };
let mut n = 0;
for &c in &b[q..] {
if c == ch {
n += 1;
} else if c != b' ' && c != b'\t' {
return false;
}
}
n >= 3
}
fn atx(rest: &str) -> bool {
RX.heading.is_match(rest)
}
fn html_start(rest: &str, in_paragraph: bool) -> Option<u8> {
let (ind, q) = indentation(rest, 0);
if ind > 3 || !rest[q..].starts_with('<') {
return None;
}
let t = &rest[q..];
if RX.html_literals[0].0.is_match(rest) {
return Some(1);
}
if t.starts_with("<!--") {
return Some(2);
}
if t.starts_with("<?") {
return Some(3);
}
if t.starts_with("<![CDATA[") {
return Some(5);
}
if RX.html_literals[3].0.is_match(rest) {
return Some(4);
}
if RX.html_block.is_match(rest) {
return Some(6);
}
if !in_paragraph && RX.html_tag_line.is_match(rest) {
return Some(7);
}
None
}
fn html_ends(kind: u8, line: &str) -> bool {
match kind {
1 => RX.html_literals[0].1.is_match(line),
2 => line.contains("-->"),
3 => line.contains("?>"),
4 => line.contains('>'),
5 => line.contains("]]>"),
_ => false,
}
}
fn delimiter_row(rest: &str) -> Option<usize> {
let t = rest.trim_matches([' ', '\t']);
if t.is_empty() || !t.contains('-') {
return None;
}
let t = t.strip_prefix('|').unwrap_or(t);
let t = t.strip_suffix('|').unwrap_or(t);
let mut n = 0;
for cell in t.split('|') {
let c = cell.trim_matches([' ', '\t']);
let c = c.strip_prefix(':').unwrap_or(c);
let c = c.strip_suffix(':').unwrap_or(c);
if c.is_empty() || !c.bytes().all(|b| b == b'-') {
return None;
}
n += 1;
}
if !rest.contains('|') && n == 1 {
return None;
}
Some(n)
}
fn row_cells(line: &str, p: usize, base: usize) -> Vec<(usize, usize)> {
let b = line.as_bytes();
let mut start = p;
let mut end = b.len();
while start < end && (b[start] == b' ' || b[start] == b'\t') {
start += 1;
}
while end > start && (b[end - 1] == b' ' || b[end - 1] == b'\t') {
end -= 1;
}
if start < end && b[start] == b'|' {
start += 1;
}
if end > start && b[end - 1] == b'|' && !(end >= 2 + start && b[end - 2] == b'\\') {
end -= 1;
}
let mut cells = Vec::new();
let mut cell_start = start;
let mut i = start;
while i < end {
if b[i] == b'\\' {
i += 2;
continue;
}
if b[i] == b'|' {
cells.push((cell_start, i));
cell_start = i + 1;
}
i += 1;
}
cells.push((cell_start, end.max(cell_start)));
cells
.into_iter()
.map(|(a, z)| {
let (mut a, mut z) = (a, z.min(b.len()));
while a < z && (b[a] == b' ' || b[a] == b'\t') {
a += 1;
}
while z > a && (b[z - 1] == b' ' || b[z - 1] == b'\t') {
z -= 1;
}
(base + a, base + z)
})
.collect()
}
fn header_cells(line: &str) -> usize {
row_cells(line, 0, 0).len()
}
fn definition(rest: &str) -> Option<String> {
let (ind, q) = indentation(rest, 0);
if ind > 3 {
return None;
}
let b = rest.as_bytes();
if b.get(q) != Some(&b'[') {
return None;
}
let mut k = q + 1;
while k < b.len() && b[k] != b']' {
if b[k] == b'[' {
return None;
}
k += if b[k] == b'\\' { 2 } else { 1 };
}
if k >= b.len() || b.get(k + 1) != Some(&b':') {
return None;
}
let label = &rest[q + 1..k];
if label.trim().is_empty() || label.len() > 999 {
return None;
}
let mut i = k + 2;
while i < b.len() && (b[i] == b' ' || b[i] == b'\t') {
i += 1;
}
if i >= b.len() {
return None;
}
if b[i] == b'<' {
i += 1;
while i < b.len() && b[i] != b'>' {
if b[i] == b'<' {
return None;
}
i += if b[i] == b'\\' { 2 } else { 1 };
}
if i >= b.len() {
return None;
}
i += 1;
} else {
let mut depth = 0i32;
let start = i;
while i < b.len() && b[i] > b' ' {
match b[i] {
b'\\' => i += 1,
b'(' => depth += 1,
b')' => {
if depth == 0 {
break;
}
depth -= 1;
}
_ => {}
}
i += 1;
}
if i == start || depth != 0 {
return None;
}
}
let after_dest = i;
while i < b.len() && (b[i] == b' ' || b[i] == b'\t') {
i += 1;
}
if i < b.len() {
if i == after_dest {
return None;
}
let close = match b[i] {
b'"' => b'"',
b'\'' => b'\'',
b'(' => b')',
_ => return None,
};
i += 1;
while i < b.len() && b[i] != close {
i += if b[i] == b'\\' { 2 } else { 1 };
}
if i >= b.len() {
return None;
}
if !is_blank(&rest[i + 1..]) {
return None;
}
}
Some(normalize_label(label))
}
fn normalize_label(s: &str) -> String {
s.split_whitespace().collect::<Vec<_>>().join(" ").to_lowercase()
}
impl Blocks {
fn read(source: &str) -> Blocks {
let mut b = Blocks { chunks: Vec::new(), defs: HashSet::new() };
let mut containers: Vec<Container> = Vec::new();
let mut leaf = Leaf::None;
let mut para: Vec<(usize, usize)> = Vec::new();
let mut table_cols = 0;
let mut offset = 0;
let bom = if source.starts_with('\u{feff}') { 3 } else { 0 };
for raw_line in source.split_inclusive('\n') {
let line_start = offset;
offset += raw_line.len();
let line = raw_line.trim_end_matches('\n').trim_end_matches('\r');
let skip = if line_start == 0 { bom.min(line.len()) } else { 0 };
let line_base = line_start;
let mut p = skip;
let mut matched = 0;
for c in &containers {
match c {
Container::Quote => {
let (ind, q) = indentation(line, p);
if ind <= 3 && line.as_bytes().get(q) == Some(&b'>') {
p = q + 1;
if matches!(line.as_bytes().get(p), Some(b' ' | b'\t')) {
p += 1;
}
matched += 1;
} else {
break;
}
}
Container::Item(width) => {
if is_blank(&line[p..]) {
matched += 1;
} else if indentation(line, p).0 >= *width {
p = consume_columns(line, p, *width);
matched += 1;
} else {
break;
}
}
}
}
let all = matched == containers.len();
if all {
if let Leaf::Fenced(ch, len) = leaf {
if fence_close(line, p, ch, len) {
leaf = Leaf::None;
}
continue;
}
if let Leaf::Html(kind) = leaf {
if kind <= 5 {
if html_ends(kind, &line[p..]) {
leaf = Leaf::None;
}
continue;
}
if is_blank(&line[p..]) {
leaf = Leaf::None;
} else {
continue;
}
}
}
let rest = &line[p..];
if !all && leaf == Leaf::Paragraph && !is_blank(rest) && !Self::interrupts(rest, true) {
para.push(Self::text_span(line, p, line_base));
continue;
}
if !all {
b.finish(&mut leaf, &mut para);
containers.truncate(matched);
}
loop {
let rest = &line[p..];
let (ind, q) = indentation(line, p);
if ind > 3 {
break;
}
if line.as_bytes().get(q) == Some(&b'>') {
b.finish(&mut leaf, &mut para);
containers.push(Container::Quote);
p = q + 1;
if matches!(line.as_bytes().get(p), Some(b' ' | b'\t')) {
p += 1;
}
continue;
}
if thematic(rest) {
break;
}
if let Some((width, ordered, one, blank_after, content)) = list_marker(line, p) {
if leaf == Leaf::Paragraph && (blank_after || ordered && !one) {
break;
}
b.finish(&mut leaf, &mut para);
containers.push(Container::Item(width));
p = content;
continue;
}
break;
}
let rest = &line[p..];
if is_blank(rest) {
if leaf != Leaf::Indented {
b.finish(&mut leaf, &mut para);
}
continue;
}
let (ind, _) = indentation(line, p);
if leaf == Leaf::Indented {
if ind >= 4 {
continue;
}
leaf = Leaf::None;
}
if leaf == Leaf::Table {
if Self::interrupts(rest, false) {
leaf = Leaf::None;
} else {
for span in row_cells(line, p, line_base) {
b.chunks.push(Chunk { spans: vec![span], cell: true });
}
continue;
}
}
if ind >= 4 && leaf != Leaf::Paragraph {
leaf = Leaf::Indented;
continue;
}
if let Some((ch, len)) = fence_open(line, p) {
b.finish(&mut leaf, &mut para);
leaf = Leaf::Fenced(ch, len);
continue;
}
if let Some(kind) = html_start(rest, leaf == Leaf::Paragraph) {
b.finish(&mut leaf, &mut para);
leaf = Leaf::Html(kind);
if kind <= 5 && html_ends(kind, rest) {
leaf = Leaf::None;
}
continue;
}
if atx(rest) {
b.finish(&mut leaf, &mut para);
b.chunks.push(Chunk { spans: vec![Self::heading_span(line, p, line_base)], cell: false });
continue;
}
if leaf == Leaf::Paragraph && Self::setext(rest) {
b.finish(&mut leaf, &mut para);
continue;
}
if thematic(rest) {
b.finish(&mut leaf, &mut para);
continue;
}
if leaf == Leaf::Paragraph
&& para.len() == 1
&& let Some(n) = delimiter_row(rest)
{
let (a, z) = para[0];
if header_cells(&source[a..z]) == n {
for span in row_cells(&source[a..z], 0, a) {
b.chunks.push(Chunk { spans: vec![span], cell: true });
}
para.clear();
leaf = Leaf::Table;
table_cols = n;
continue;
}
}
if leaf != Leaf::Paragraph {
leaf = Leaf::Paragraph;
para.clear();
}
if para.is_empty()
&& let Some(label) = definition(rest)
{
b.defs.insert(label);
continue;
}
para.push(Self::text_span(line, p, line_base));
}
let _ = table_cols;
b.finish(&mut leaf, &mut para);
b
}
fn text_span(line: &str, p: usize, base: usize) -> (usize, usize) {
let (_, q) = indentation(line, p);
(base + q, base + line.len())
}
fn heading_span(line: &str, p: usize, base: usize) -> (usize, usize) {
let b = line.as_bytes();
let (_, mut q) = indentation(line, p);
while q < b.len() && b[q] == b'#' {
q += 1;
}
while q < b.len() && (b[q] == b' ' || b[q] == b'\t') {
q += 1;
}
let mut end = b.len();
while end > q && (b[end - 1] == b' ' || b[end - 1] == b'\t') {
end -= 1;
}
let mut k = end;
while k > q && b[k - 1] == b'#' {
k -= 1;
}
if k < end && (k == q || b[k - 1] == b' ' || b[k - 1] == b'\t') {
end = k;
while end > q && (b[end - 1] == b' ' || b[end - 1] == b'\t') {
end -= 1;
}
}
(base + q, base + end.max(q))
}
fn setext(rest: &str) -> bool {
let (ind, q) = indentation(rest, 0);
if ind > 3 {
return false;
}
let t = rest[q..].trim_end_matches([' ', '\t']);
!t.is_empty() && (t.bytes().all(|c| c == b'=') || t.bytes().all(|c| c == b'-'))
}
fn interrupts(rest: &str, paragraph: bool) -> bool {
let (ind, q) = indentation(rest, 0);
if ind > 3 {
return false;
}
let b = rest.as_bytes();
b.get(q) == Some(&b'>')
|| atx(rest)
|| thematic(rest)
|| fence_open(rest, 0).is_some()
|| html_start(rest, paragraph).is_some()
|| list_marker(rest, 0).is_some_and(|(_, ordered, one, blank_after, _)| !blank_after && (!ordered || one))
}
fn finish(&mut self, leaf: &mut Leaf, para: &mut Vec<(usize, usize)>) {
if *leaf == Leaf::Paragraph && !para.is_empty() {
self.chunks.push(Chunk { spans: std::mem::take(para), cell: false });
}
para.clear();
if matches!(*leaf, Leaf::Paragraph | Leaf::Table) {
*leaf = Leaf::None;
}
}
}
struct LabelStart {
image: bool,
inactive: bool,
text: usize,
}
struct Scanner<'a> {
v: &'a str,
map: &'a [usize],
source: &'a str,
defs: &'a HashSet<String>,
out: &'a mut Vec<InlineToken>,
cell: bool,
closers: HashMap<usize, usize>,
math_fails_until: usize,
}
fn label_closers(v: &str) -> HashMap<usize, usize> {
let b = v.as_bytes();
let mut out = HashMap::new();
let mut open: Vec<usize> = Vec::new();
let mut i = 0;
while i < b.len() {
match b[i] {
b'\\' if i + 1 < b.len() && matches!(b[i + 1], b'[' | b']' | b'\\') => i += 1,
b'[' => open.push(i),
b']' => {
if let Some(o) = open.pop() {
out.insert(o, i);
}
}
b'\n' | b'\r' => open.clear(),
_ => {}
}
i += 1;
}
out
}
fn byte(s: &str, i: usize) -> u8 {
s.as_bytes().get(i).copied().unwrap_or(0)
}
fn spaced(prev: Option<char>) -> bool {
matches!(prev, None | Some(' ' | '\t' | '\n' | '\r'))
}
fn ascii_punctuation(c: u8) -> bool {
c.is_ascii_punctuation()
}
impl Scanner<'_> {
fn src(&self, i: usize) -> usize {
self.map.get(i).copied().unwrap_or(0)
}
fn scan(&mut self, from: usize, to: usize, depth: usize) {
let v = self.v;
let b = v.as_bytes();
let mut i = from;
let mut prev: Option<char> = None;
let mut labels: Vec<LabelStart> = Vec::new();
let mut active = 0usize;
while i < to {
let c = b[i];
let at = i;
let next = match c {
b'\\' => {
if i + 1 < to && ascii_punctuation(b[i + 1]) {
Some((i + 2, b[i + 1] as char))
} else {
None
}
}
b'`' => self.code_span(i, to),
b'<' => self.autolink(i, to).or_else(|| self.html(i, to)),
b':' if spaced(prev) || matches!(prev, Some('(' | '[' | '{' | '<' | '"' | '\'')) => self.directive(i, to, depth),
b'[' => {
if let Some(n) = self.footnote(i, to).or_else(|| self.wiki(i, to, false)) {
Some(n)
} else {
labels.push(LabelStart { image: false, inactive: false, text: i + 1 });
active += 1;
Some((i + 1, '['))
}
}
b'!' => {
if let Some(n) = self.wiki(i, to, true) {
Some(n)
} else if byte(v, i + 1) == b'[' && i + 1 < to {
labels.push(LabelStart { image: true, inactive: false, text: i + 2 });
active += 1;
Some((i + 2, '['))
} else {
None
}
}
b']' => self.label_end(i, to, &mut labels, &mut active),
b'$' => self.param(i, to).or_else(|| self.math(i, to)),
b'h' | b'H' if !prev.is_some_and(|p| p.is_ascii_alphabetic()) && active == 0 => self.protocol_autolink(i, to),
b'w' | b'W' if (spaced(prev) || matches!(prev, Some('(' | '*' | '_' | '[' | ']' | '~'))) && active == 0 => {
self.www_autolink(i, to)
}
_ => None,
};
match next {
Some((n, last)) if n > at => {
i = n;
prev = Some(last);
}
_ => {
let ch = v[i..].chars().next().unwrap_or(' ');
i += ch.len_utf8();
prev = Some(ch);
}
}
}
}
fn code_span(&self, i: usize, to: usize) -> Option<(usize, char)> {
let b = self.v.as_bytes();
let mut n = 0;
while i + n < to && b[i + n] == b'`' {
n += 1;
}
let mut k = i + n;
while k < to {
if b[k] == b'`' {
let mut m = 0;
while k + m < to && b[k + m] == b'`' {
m += 1;
}
if m == n {
return Some((k + m, '`'));
}
k += m;
} else {
k += 1;
}
}
Some((i + n, '`'))
}
fn autolink(&self, i: usize, to: usize) -> Option<(usize, char)> {
let b = self.v.as_bytes();
let mut k = i + 1;
let start = k;
while k < to && (b[k].is_ascii_alphanumeric() || matches!(b[k], b'+' | b'.' | b'-')) {
k += 1;
}
if k - start >= 2 && k - start <= 32 && b[start].is_ascii_alphabetic() && k < to && b[k] == b':' {
k += 1;
while k < to && b[k] > b' ' && b[k] != b'<' && b[k] != b'>' && b[k] != 0x7f {
k += 1;
}
if k < to && b[k] == b'>' {
return Some((k + 1, '>'));
}
return None;
}
let mut k = i + 1;
let local = k;
while k < to && (b[k].is_ascii_alphanumeric() || b".!#$%&'*+/=?^_`{|}~-".contains(&b[k])) {
k += 1;
}
if k == local || k >= to || b[k] != b'@' {
return None;
}
k += 1;
loop {
let label = k;
while k < to && (b[k].is_ascii_alphanumeric() || b[k] == b'-') && k - label < 63 {
k += 1;
}
if k == label {
return None;
}
if k < to && b[k] == b'.' {
k += 1;
continue;
}
break;
}
if k < to && b[k] == b'>' { Some((k + 1, '>')) } else { None }
}
fn html(&self, i: usize, to: usize) -> Option<(usize, char)> {
let v = &self.v[..to];
let b = v.as_bytes();
let rest = &v[i..];
let find = |pat: &str, from: usize| v[from..].find(pat).map(|p| from + p);
if rest.starts_with("<!--") {
if rest.starts_with("<!-->") {
return Some((i + 5, '>'));
}
if rest.starts_with("<!--->") {
return Some((i + 6, '>'));
}
return find("-->", i + 4).map(|e| (e + 3, '>'));
}
if rest.starts_with("<?") {
return find("?>", i + 2).map(|e| (e + 2, '>'));
}
if rest.starts_with("<![CDATA[") {
return find("]]>", i + 9).map(|e| (e + 3, '>'));
}
if rest.starts_with("<!") {
if !byte(v, i + 2).is_ascii_alphabetic() {
return None;
}
return find(">", i + 2).map(|e| (e + 1, '>'));
}
let ws = |c: u8| matches!(c, b' ' | b'\t' | b'\n' | b'\r');
if rest.starts_with("</") {
let mut k = i + 2;
if !byte(v, k).is_ascii_alphabetic() {
return None;
}
while k < b.len() && (b[k].is_ascii_alphanumeric() || b[k] == b'-') {
k += 1;
}
while k < b.len() && ws(b[k]) {
k += 1;
}
return (byte(v, k) == b'>').then_some((k + 1, '>'));
}
let mut k = i + 1;
if !byte(v, k).is_ascii_alphabetic() {
return None;
}
while k < b.len() && (b[k].is_ascii_alphanumeric() || b[k] == b'-') {
k += 1;
}
loop {
let before = k;
while k < b.len() && ws(b[k]) {
k += 1;
}
match byte(v, k) {
b'>' => return Some((k + 1, '>')),
b'/' if byte(v, k + 1) == b'>' => return Some((k + 2, '>')),
c if (c.is_ascii_alphabetic() || c == b'_' || c == b':') && k > before => {
while k < b.len() && (b[k].is_ascii_alphanumeric() || matches!(b[k], b'_' | b'.' | b':' | b'-')) {
k += 1;
}
let save = k;
while k < b.len() && ws(b[k]) {
k += 1;
}
if byte(v, k) == b'=' {
k += 1;
while k < b.len() && ws(b[k]) {
k += 1;
}
match byte(v, k) {
q @ (b'"' | b'\'') => {
let close = v[k + 1..].find(q as char)?;
k = k + 1 + close + 1;
}
_ => {
let s = k;
while k < b.len() && !ws(b[k]) && !b"\"'=<>`".contains(&b[k]) {
k += 1;
}
if k == s {
return None;
}
}
}
} else {
k = save;
}
}
_ => return None,
}
}
}
fn directive(&mut self, i: usize, to: usize, depth: usize) -> Option<(usize, char)> {
let v = self.v;
let b = v.as_bytes();
let mut k = i + 1;
if !(k < to && b[k].is_ascii_lowercase()) {
return None;
}
let name_start = k;
let mut qualified = false;
loop {
while k < to && (b[k].is_ascii_lowercase() || b[k].is_ascii_digit() || b[k] == b'-') {
k += 1;
}
if k < to && b[k] == b':' && !qualified {
qualified = true;
k += 1;
if !(k < to && b[k].is_ascii_lowercase()) {
return None;
}
continue;
}
break;
}
if !(k < to && b[k] == b'[') {
return None;
}
let name_end = k;
let label_open = k;
let label_start = k + 1;
k = match self.closers.get(&label_open) {
Some(&close) if close < to => close,
_ => return None,
};
let label_end = k;
k += 1;
let mut end = k;
let mut attributes = None;
if k < to
&& b[k] == b'{'
&& let Some(close) = Self::inline_attributes(v, k, to)
{
attributes = Some((k + 1, close - 1));
end = close;
}
let mut t = InlineToken::new(TokenType::Directive, self.src(i), self.src(end));
t.kind = self.source[self.src(name_start)..self.src(name_end)].to_string();
if label_end > label_start {
t.label_start = self.src(label_start);
t.label_end = self.src(label_end);
t.label = self.source[t.label_start..t.label_end].to_string();
} else {
t.label_start = self.src(label_open) + 1;
t.label_end = t.label_start;
}
if let Some((a, z)) = attributes {
t.attributes = Some(self.source[self.src(a)..self.src(z)].to_string());
}
self.out.push(t);
if label_end > label_start && depth < MAX_LABEL_DEPTH {
self.scan(label_start, label_end, depth + 1);
}
let last = if attributes.is_some() { '}' } else { ']' };
Some((end, last))
}
fn inline_attributes(v: &str, k: usize, to: usize) -> Option<usize> {
let b = v.as_bytes();
let mut j = k + 1;
let mut quoted = false;
while j < to {
let c = b[j];
if c == b'\n' || c == b'\r' {
return None;
}
if quoted {
if c == b'\\' {
if j + 1 >= to || b[j + 1] == b'\n' || b[j + 1] == b'\r' {
return None;
}
j += 2;
continue;
}
if c == b'"' {
quoted = false;
}
j += 1;
continue;
}
match c {
b'{' => return None,
b'"' => quoted = true,
b'}' => return Some(j + 1),
_ => {}
}
j += 1;
}
None
}
fn footnote(&mut self, i: usize, to: usize) -> Option<(usize, char)> {
let b = self.v.as_bytes();
if !(i + 1 < to && b[i + 1] == b'^') {
return None;
}
let mut k = i + 2;
if !(k < to && b[k].is_ascii_alphanumeric()) {
return None;
}
while k < to && (b[k].is_ascii_alphanumeric() || matches!(b[k], b'.' | b'_' | b'-')) {
k += 1;
}
if !(k < to && b[k] == b']') {
return None;
}
k += 1;
if k < to && (b[k] == b'(' || b[k] == b':') {
return None;
}
let (s, e) = (self.src(i), self.src(k));
let mut t = InlineToken::new(TokenType::Footnote, s, e);
t.label = self.source[s + 2..e - 1].to_string();
self.out.push(t);
Some((k, ']'))
}
fn wiki(&mut self, i: usize, to: usize, embed: bool) -> Option<(usize, char)> {
let b = self.v.as_bytes();
let open = if embed { i + 1 } else { i };
if !(open + 1 < to && b[open] == b'[' && b[open + 1] == b'[') || embed && b[i] != b'!' {
return None;
}
let mut k = open + 2;
let mut empty = true;
loop {
if k >= to || b[k] == b'\n' || b[k] == b'\r' || b[k] == b'[' {
return None;
}
if b[k] == b']' {
if empty {
return None;
}
break;
}
empty = false;
if b[k] == b'\\' {
k += 1;
if k >= to || b[k] == b'\n' || b[k] == b'\r' {
return None;
}
}
k += 1;
}
if !(k + 1 < to && b[k + 1] == b']') {
return None;
}
let end = k + 2;
if end < to && b[end] == b'(' {
return None;
}
let (s, e) = (self.src(i), self.src(end));
let raw = &self.source[s..e];
let mut inner = if embed { raw[3..raw.len() - 2].to_string() } else { raw[2..raw.len() - 2].to_string() };
if self.cell {
inner = inner.replace("\\|", "|");
}
let (target, label, piped) = split_wiki(&inner);
let mut t = InlineToken::new(if embed { TokenType::Embed } else { TokenType::Wiki }, s, e);
t.target = target;
t.label = label;
t.piped = piped;
self.out.push(t);
Some((end, ']'))
}
fn param(&mut self, i: usize, to: usize) -> Option<(usize, char)> {
let b = self.v.as_bytes();
if !(i + 1 < to && b[i + 1] == b'{') {
return None;
}
let mut k = i + 2;
if !(k < to && (b[k].is_ascii_alphabetic() || b[k] == b'_')) {
return None;
}
while k < to && (b[k].is_ascii_alphanumeric() || matches!(b[k], b'_' | b'.' | b'-')) {
k += 1;
}
if !(k < to && b[k] == b'}') {
return None;
}
k += 1;
let (s, e) = (self.src(i), self.src(k));
let mut t = InlineToken::new(TokenType::Param, s, e);
t.name = self.source[s + 2..e - 1].to_string();
self.out.push(t);
Some((k, '}'))
}
fn math(&mut self, i: usize, to: usize) -> Option<(usize, char)> {
let v = self.v;
let b = v.as_bytes();
if i < self.math_fails_until {
return None;
}
let first = *b.get(i + 1).filter(|_| i + 1 < to)?;
if first == b' ' || first == b'\t' || first == b'\n' || first == b'\r' || first.is_ascii_digit() || first == b'$' {
return None;
}
let mut k = i + 1;
let mut previous: Option<char> = None;
let mut escaped = false;
while k < to {
let c = b[k];
if c == b'\n' || c == b'\r' {
if !escaped {
self.math_fails_until = k;
}
return None;
}
escaped = escaped || c == b'\\';
if c == b'$' && !spaced(previous) {
let after = if k + 1 < to { b[k + 1] } else { 0 };
if !after.is_ascii_digit() {
let end = k + 1;
let (s, e) = (self.src(i), self.src(end));
let mut t = InlineToken::new(TokenType::Math, s, e);
t.formula = self.source[s + 1..e - 1].to_string();
self.out.push(t);
return Some((end, '$'));
}
}
let ch = v[k..].chars().next().unwrap_or(' ');
previous = Some(ch);
k += ch.len_utf8();
if c == b'\\' {
if k >= to || b[k] == b'\n' || b[k] == b'\r' {
return None;
}
let ch = v[k..].chars().next().unwrap_or(' ');
previous = Some(ch);
k += ch.len_utf8();
}
}
if !escaped {
self.math_fails_until = to;
}
None
}
fn label_end(&mut self, i: usize, to: usize, labels: &mut Vec<LabelStart>, active: &mut usize) -> Option<(usize, char)> {
let start = labels.pop()?;
if start.inactive {
return None;
}
*active -= 1;
let v = self.v;
let b = v.as_bytes();
let after = i + 1;
let mut link_end = None;
if after < to && b[after] == b'(' {
link_end = self.resource(after, to);
}
let defined = |label: &str| label.len() <= 999 && self.defs.contains(&normalize_label(label));
if self.defs.is_empty() {
} else if link_end.is_none() && after < to && b[after] == b'[' {
let horizon = (after + 1 + 1001).min(to);
if let Some(close) = v.as_bytes()[after + 1..horizon].iter().position(|&c| c == b']').map(|p| after + 1 + p) {
let inner = &v[after + 1..close];
let label = if inner.is_empty() { &v[start.text..i] } else { inner };
if !inner.contains('[') && defined(label) {
link_end = Some(close + 1);
}
}
}
if link_end.is_none() && !self.defs.is_empty() && defined(&v[start.text..i]) {
link_end = Some(after);
}
let end = link_end?;
if !start.image {
for l in labels.iter_mut() {
if !l.image && !l.inactive {
l.inactive = true;
*active -= 1;
}
}
}
Some((end, byte(v, end - 1) as char))
}
fn resource(&self, k: usize, to: usize) -> Option<usize> {
let b = self.v.as_bytes();
let ws = |c: u8| matches!(c, b' ' | b'\t' | b'\n' | b'\r');
let mut j = k + 1;
let skip_ws = |j: &mut usize| {
let mut breaks = 0;
while *j < to && ws(b[*j]) {
if b[*j] == b'\n' {
breaks += 1;
}
*j += 1;
}
breaks <= 1
};
if !skip_ws(&mut j) {
return None;
}
if j < to && b[j] == b')' {
return Some(j + 1);
}
if j < to && b[j] == b'<' {
j += 1;
while j < to && b[j] != b'>' {
if b[j] == b'<' || b[j] == b'\n' || b[j] == b'\r' {
return None;
}
j += if b[j] == b'\\' { 2 } else { 1 };
}
if j >= to {
return None;
}
j += 1;
} else {
let mut depth = 0;
let start = j;
while j < to && b[j] > b' ' && b[j] != 0x7f {
match b[j] {
b'\\' => j += 1,
b'(' => {
depth += 1;
if depth > 32 {
return None;
}
}
b')' => {
if depth == 0 {
break;
}
depth -= 1;
}
_ => {}
}
j += 1;
}
if j == start || depth != 0 {
return None;
}
}
let before_ws = j;
if !skip_ws(&mut j) {
return None;
}
if j < to && j > before_ws && matches!(b[j], b'"' | b'\'' | b'(') {
let close = if b[j] == b'(' { b')' } else { b[j] };
j += 1;
while j < to && b[j] != close {
if b[j] == b'(' && close == b')' {
return None;
}
j += if b[j] == b'\\' { 2 } else { 1 };
}
if j >= to {
return None;
}
j += 1;
if !skip_ws(&mut j) {
return None;
}
}
if j < to && b[j] == b')' { Some(j + 1) } else { None }
}
fn protocol_autolink(&self, i: usize, to: usize) -> Option<(usize, char)> {
let v = self.v;
let b = v.as_bytes();
let mut k = i;
while k < to && b[k].is_ascii_alphabetic() && k - i < 5 {
k += 1;
}
let protocol = v.as_bytes()[i..k].to_ascii_lowercase();
if !(protocol == b"http" || protocol == b"https")
|| byte(v, k) != b':'
|| byte(v, k + 1) != b'/'
|| byte(v, k + 2) != b'/'
|| k + 3 > to
{
return None;
}
k += 3;
let c = v[k..to].chars().next()?;
if c.is_ascii_control() || c.is_whitespace() || is_unicode_punctuation(c) {
return None;
}
let k = self.domain(k, to)?;
let k = self.path(k, to);
Some((k, v[..k].chars().next_back().unwrap_or(' ')))
}
fn www_autolink(&self, i: usize, to: usize) -> Option<(usize, char)> {
let v = self.v;
if i + 4 > to || !v.as_bytes()[i..i + 3].eq_ignore_ascii_case(b"www") || byte(v, i + 3) != b'.' {
return None;
}
let k = self.domain(i, to)?;
let k = self.path(k, to);
Some((k, v[..k].chars().next_back().unwrap_or(' ')))
}
fn domain(&self, k: usize, to: usize) -> Option<usize> {
let v = self.v;
let mut j = k;
let mut seen = false;
let mut underscore_last = false;
let mut underscore_last_last = false;
while j < to {
let c = v[j..].chars().next()?;
if c == '.' || c == '_' {
if self.trail(j, to) {
break;
}
if c == '_' {
underscore_last = true;
} else {
underscore_last_last = underscore_last;
underscore_last = false;
}
j += 1;
continue;
}
if c.is_whitespace() || c != '-' && is_unicode_punctuation(c) {
break;
}
seen = true;
j += c.len_utf8();
}
if underscore_last || underscore_last_last || !seen {
return None;
}
Some(j)
}
fn path(&self, k: usize, to: usize) -> usize {
let v = self.v;
let mut j = k;
let (mut open, mut close) = (0, 0);
while j < to {
let Some(c) = v[j..].chars().next() else { break };
if c == '(' {
open += 1;
j += 1;
continue;
}
if c == ')' && close < open {
close += 1;
j += 1;
continue;
}
if matches!(c, '!' | '"' | '&' | '\'' | ')' | '*' | ',' | '.' | ':' | ';' | '<' | '?' | ']' | '_' | '~') {
if self.trail(j, to) {
break;
}
if c == ')' {
close += 1;
}
j += 1;
continue;
}
if c.is_whitespace() {
break;
}
j += c.len_utf8();
}
j
}
fn trail(&self, j: usize, to: usize) -> bool {
let v = self.v;
let b = v.as_bytes();
let mut k = j;
loop {
if k >= to {
return true;
}
let c = b[k];
match c {
b'!' | b'"' | b'\'' | b')' | b'*' | b',' | b'.' | b':' | b';' | b'?' | b'_' | b'~' => k += 1,
b'&' => {
k += 1;
if !(k < to && b[k].is_ascii_alphabetic()) {
return false;
}
while k < to && b[k].is_ascii_alphabetic() {
k += 1;
}
if !(k < to && b[k] == b';') {
return false;
}
k += 1;
}
b']' => {
k += 1;
if k >= to || matches!(b[k], b'(' | b'[' | b' ' | b'\t' | b'\n' | b'\r') {
return true;
}
let n = v[k..].chars().next().unwrap_or(' ');
if n.is_whitespace() {
return true;
}
}
b'<' | b' ' | b'\t' | b'\n' | b'\r' => return true,
_ => {
let n = v[k..].chars().next().unwrap_or(' ');
return n.is_whitespace();
}
}
}
}
}
fn is_unicode_punctuation(c: char) -> bool {
if c.is_ascii() {
return c.is_ascii_punctuation();
}
!c.is_alphanumeric() && !c.is_whitespace() && !matches!(c, '\u{300}'..='\u{36f}')
}