mbdown 0.1.0

//! Lexical MBDown extension tokens. Backends decide how semantic constructs
//! map to their native style representation.

#[derive(Clone, Debug, PartialEq, Eq)]
pub enum InlineToken<'a> {
    Text(&'a str),
    Hashtag(&'a str),
    WikiLink(&'a str),
    Tag {
        raw: &'a str,
        name: String,
        value: Option<String>,
        closing: bool,
    },
}

pub fn tokenize_inline(input: &str) -> Vec<InlineToken<'_>> {
    let mut tokens = Vec::new();
    let mut cursor = 0;
    let mut text_start = 0;
    while cursor < input.len() {
        let ch = input[cursor..]
            .chars()
            .next()
            .expect("cursor is on a character boundary");
        if ch == '#' {
            if let Some(end) = hashtag_end(input, cursor) {
                push_text(&mut tokens, input, text_start, cursor);
                tokens.push(InlineToken::Hashtag(&input[cursor + 1..end]));
                cursor = end;
                text_start = cursor;
                continue;
            }
        } else if ch == '[' && input[cursor..].starts_with("[[") {
            if let Some(end) = wikilink_end(input, cursor) {
                push_text(&mut tokens, input, text_start, cursor);
                tokens.push(InlineToken::WikiLink(&input[cursor + 2..end]));
                cursor = end + 2;
                text_start = cursor;
                continue;
            }
        }
        if ch != '[' {
            cursor += ch.len_utf8();
            continue;
        }
        let Some(end) = find_tag_end(input, cursor + 1) else {
            cursor += 1;
            continue;
        };
        let body = input[cursor + 1..end].trim();
        let Some((name, value, closing)) = parse_tag(body) else {
            cursor = end + 1;
            continue;
        };
        push_text(&mut tokens, input, text_start, cursor);
        tokens.push(InlineToken::Tag {
            raw: &input[cursor..=end],
            name,
            value,
            closing,
        });
        cursor = end + 1;
        text_start = cursor;
    }
    if text_start < input.len() {
        tokens.push(InlineToken::Text(&input[text_start..]));
    }
    tokens
}

fn push_text<'a>(tokens: &mut Vec<InlineToken<'a>>, input: &'a str, start: usize, end: usize) {
    if start < end {
        tokens.push(InlineToken::Text(&input[start..end]));
    }
}

fn hashtag_end(input: &str, start: usize) -> Option<usize> {
    if input[..start]
        .chars()
        .next_back()
        .is_some_and(|character| character.is_alphanumeric() || character == '_')
    {
        return None;
    }
    let mut end = start + 1;
    let first = input.get(end..)?.chars().next()?;
    if !(first.is_alphanumeric() || first == '_') {
        return None;
    }
    end += first.len_utf8();
    for character in input[end..].chars() {
        if character.is_alphanumeric() || matches!(character, '_' | '-' | '/') {
            end += character.len_utf8();
        } else {
            break;
        }
    }
    Some(end)
}

fn wikilink_end(input: &str, start: usize) -> Option<usize> {
    let body_start = start + 2;
    let relative_end = input.get(body_start..)?.find("]]")?;
    let end = body_start + relative_end;
    let body = &input[body_start..end];
    (!body.is_empty() && body.trim() == body && !body.contains(['\n', '\r', '['])).then_some(end)
}

fn parse_tag(body: &str) -> Option<(String, Option<String>, bool)> {
    if body.is_empty() {
        return None;
    }
    let (closing, body) = body
        .strip_prefix('/')
        .map_or((false, body), |rest| (true, rest.trim()));
    let (name, value) = body.split_once('=').map_or((body, None), |(name, value)| {
        (name.trim(), parse_value(value.trim()))
    });
    let name = name.trim().to_ascii_lowercase();
    if (!closing && name.is_empty())
        || !name
            .bytes()
            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
        || (closing && value.is_some())
    {
        return None;
    }
    Some((name, value, closing))
}

fn parse_value(value: &str) -> Option<String> {
    if !value.starts_with('"') {
        return Some(value.to_string());
    }
    let mut output = String::new();
    let mut escaped = false;
    let mut chars = value[1..].chars();
    while let Some(ch) = chars.next() {
        if escaped {
            output.push(ch);
            escaped = false;
        } else if ch == '\\' {
            escaped = true;
        } else if ch == '"' {
            return chars.as_str().trim().is_empty().then_some(output);
        } else {
            output.push(ch);
        }
    }
    None
}

fn find_tag_end(input: &str, start: usize) -> Option<usize> {
    let mut quoted = false;
    let mut escaped = false;
    for (offset, ch) in input.get(start..)?.char_indices() {
        if escaped {
            escaped = false;
        } else if quoted && ch == '\\' {
            escaped = true;
        } else if ch == '"' {
            quoted = !quoted;
        } else if ch == ']' && !quoted {
            return Some(start + offset);
        }
    }
    None
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn tokenizes_styles_and_quoted_links() {
        let tokens = tokenize_inline("a[red]b[/red][link=\"A]B\"]c[/link]");
        assert_eq!(tokens.len(), 7);
        assert!(matches!(
            &tokens[4],
            InlineToken::Tag {
                name,
                value: Some(value),
                ..
            } if name == "link" && value == "A]B"
        ));
    }

    #[test]
    fn tokenizes_hashtags_and_wikilinks() {
        assert_eq!(
            tokenize_inline("See #rust/lang and [[Rust Notes]]."),
            vec![
                InlineToken::Text("See "),
                InlineToken::Hashtag("rust/lang"),
                InlineToken::Text(" and "),
                InlineToken::WikiLink("Rust Notes"),
                InlineToken::Text("."),
            ]
        );
    }

    #[test]
    fn hashtags_require_a_word_boundary_and_name() {
        assert_eq!(
            tokenize_inline("title#fragment # #中文 #开发-日志"),
            vec![
                InlineToken::Text("title#fragment # "),
                InlineToken::Hashtag("中文"),
                InlineToken::Text(" "),
                InlineToken::Hashtag("开发-日志"),
            ]
        );
    }

    #[test]
    fn invalid_wikilinks_remain_text() {
        assert_eq!(
            tokenize_inline("[[]] [[ spaced ]] [[open"),
            vec![InlineToken::Text("[[]] [[ spaced ]] [[open")]
        );
    }
}