use crate::{
error::{Error, Fix, SourceRange},
escaping::{invalid_escapes, unescape, unescaped_characters},
format::CodeStr,
line_index::LineIndex,
scoring::populate_traversal_order,
wiki::{FILESYSTEM_LINK_PREFIX, FilesystemTarget, Link, Page, Wiki, title_line_text},
};
use std::{iter, path::Path};
struct PendingPage {
title: Option<(String, SourceRange)>,
source_start: usize,
content_start: usize,
}
pub fn parse(
source_path: Option<&Path>,
source_contents: &str,
line_index: &LineIndex,
) -> (Wiki, Vec<Error>) {
let mut wiki = Wiki::new(source_contents);
let mut errors = Vec::<Error>::new();
let mut pending_page = PendingPage {
title: None,
source_start: 0,
content_start: 0,
};
let mut has_seen_title_marker = false;
let mut reported_content_before_title = false;
let mut line_start = 0;
for raw_line in source_contents.split_inclusive('\n') {
let next_line_start = line_start + raw_line.len();
let line_with_possible_carriage_return = raw_line.strip_suffix('\n').unwrap_or(raw_line);
let line = line_with_possible_carriage_return
.strip_suffix('\r')
.unwrap_or(line_with_possible_carriage_return);
let line_source_range = SourceRange {
start: line_start,
end: line_start + line.len(),
};
if let Some(raw_title) = title_line_text(line) {
has_seen_title_marker = true;
errors.extend(finish_page(
&mut wiki,
pending_page,
line_start,
source_path,
source_contents,
line_index,
));
let title = match parse_title(
raw_title,
source_path,
source_contents,
line_index,
line_source_range,
) {
Ok(title) => Some(title),
Err(error) => {
errors.push(error);
None
}
};
pending_page = PendingPage {
title,
source_start: line_start,
content_start: next_line_start,
};
} else if !has_seen_title_marker
&& !reported_content_before_title
&& !line.trim().is_empty()
{
errors.push(Error::new(
"This content isn't in any page.",
source_path,
Some((
source_contents,
line_index,
trim_source_range(source_contents, line_source_range),
)),
None,
None,
));
reported_content_before_title = true;
}
line_start = next_line_start;
}
errors.extend(finish_page(
&mut wiki,
pending_page,
source_contents.len(),
source_path,
source_contents,
line_index,
));
populate_traversal_order(&mut wiki);
(wiki, errors)
}
fn parse_title(
raw_title: &str,
source_path: Option<&Path>,
source_contents: &str,
line_index: &LineIndex,
line_source_range: SourceRange,
) -> Result<(String, SourceRange), Error> {
let title_source_range = trim_source_range(
source_contents,
SourceRange {
start: line_source_range.end - raw_title.len(),
end: line_source_range.end,
},
);
let title = &source_contents[title_source_range.start..title_source_range.end];
if title.is_empty() {
Err(Error::new(
"A page title can't be empty.",
source_path,
Some((source_contents, line_index, line_source_range)),
None,
None,
))
} else if !title.starts_with(FILESYSTEM_LINK_PREFIX) {
Ok((
title.to_owned(),
SourceRange {
start: title_source_range.start,
end: line_source_range.end,
},
))
} else {
Err(Error::new(
&format!(
"A page title can't start with {}.",
FILESYSTEM_LINK_PREFIX.code_str(),
),
source_path,
Some((source_contents, line_index, title_source_range)),
None,
None,
))
}
}
fn finish_page(
wiki: &mut Wiki,
pending_page: PendingPage,
source_end: usize,
source_path: Option<&Path>,
source_contents: &str,
line_index: &LineIndex,
) -> Vec<Error> {
let PendingPage {
title,
source_start,
content_start,
} = pending_page;
let source_range = trim_source_range(
source_contents,
SourceRange {
start: source_start,
end: source_end,
},
);
let content_source_range = trim_source_range(
source_contents,
SourceRange {
start: content_start,
end: source_end,
},
);
let (links, content_errors) = parse_content(
source_path,
source_contents,
line_index,
content_source_range,
);
let Some((title, title_source_range)) = title else {
return content_errors;
};
if wiki.pages.contains_key(&title) {
return iter::once(Error::new(
&format!("Page {} already exists.", title.code_str()),
source_path,
Some((source_contents, line_index, title_source_range)),
None,
None,
))
.chain(content_errors)
.collect();
}
wiki.pages.insert(
title.clone(),
Page {
title,
links,
traversal_index: None,
source_range,
title_source_range,
},
);
content_errors
}
#[allow(
clippy::too_many_lines,
reason = "The function scans content once, tracking the state of the link being read."
)]
fn parse_content(
source_path: Option<&Path>,
source_contents: &str,
line_index: &LineIndex,
source_range: SourceRange,
) -> (Vec<Link>, Vec<Error>) {
let original_content = &source_contents[source_range.start..source_range.end];
let mut links = Vec::<Link>::new();
let mut errors = Vec::<Error>::new();
let mut link_start = None::<usize>;
let mut link_has_line_break = false;
let mut link_has_nested_start = false;
let syntax_error = |message: &str, error_source_range| {
Error::new(
message,
source_path,
Some((source_contents, line_index, error_source_range)),
None,
None,
)
};
for (index, character) in unescaped_characters(original_content) {
let character_source_range = SourceRange {
start: source_range.start + index,
end: source_range.start + index + character.len_utf8(),
};
if character == '\n'
&& let Some(start) = link_start
&& !link_has_line_break
{
let link_source_range = SourceRange {
start: source_range.start + start,
end: source_range.start + index + character.len_utf8(),
};
errors.push(syntax_error(
"A link can't contain a line break.",
link_source_range,
));
link_has_line_break = true;
}
match character {
'[' if link_start.is_some() => {
errors.push(syntax_error(
"Unexpected opening link delimiter.",
character_source_range,
));
link_has_nested_start = true;
}
'[' => {
link_start = Some(index);
link_has_line_break = false;
link_has_nested_start = false;
}
']' if link_start.is_none() => {
errors.push(syntax_error(
"Unexpected closing link delimiter.",
character_source_range,
));
}
']' => {
let start = link_start.take().expect("The link start was checked.");
let inner_start = start + '['.len_utf8();
let trimmed_target = original_content[inner_start..index].trim();
let link_source_range = SourceRange {
start: source_range.start + start,
end: source_range.start + index + character.len_utf8(),
};
let link = if link_has_line_break || link_has_nested_start {
None
} else {
unescape(trimmed_target).map(|unescaped_target| {
parse_link(
&unescaped_target,
source_path,
source_contents,
line_index,
link_source_range,
)
})
};
match link {
Some(Ok(link)) => links.push(link),
Some(Err(error)) => errors.push(error),
None => {}
}
}
_ => {}
}
}
if let Some(start) = link_start {
let link_source_range = SourceRange {
start: source_range.start + start,
end: source_range.start + start + '['.len_utf8(),
};
errors.push(syntax_error("Unclosed link.", link_source_range));
}
for index in invalid_escapes(original_content) {
let offset = source_range.start + index;
errors.push(Error::new(
"A backslash must be followed by `[`, `]`, or `\\`, so write `\\\\` for a backslash \
itself.",
source_path,
Some((
source_contents,
line_index,
SourceRange {
start: offset,
end: offset + '\\'.len_utf8(),
},
)),
None,
Some(Fix::EscapeBackslash(offset)),
));
}
(links, errors)
}
fn parse_link(
target: &str,
source_path: Option<&Path>,
source_contents: &str,
line_index: &LineIndex,
source_range: SourceRange,
) -> Result<Link, Error> {
if target.starts_with(FILESYSTEM_LINK_PREFIX) {
let target = FilesystemTarget::parse(target).map_err(|message| {
Error::new(
&message,
source_path,
Some((source_contents, line_index, source_range)),
None,
None,
)
})?;
Ok(Link::Filesystem {
target,
source_range,
})
} else {
Ok(Link::Text {
title: target.to_owned(),
source_range,
})
}
}
fn trim_source_range(source_contents: &str, source_range: SourceRange) -> SourceRange {
let source = &source_contents[source_range.start..source_range.end];
let start_trimmed = source.trim_start();
let start = source_range.start + source.len() - start_trimmed.len();
let trimmed = start_trimmed.trim_end();
SourceRange {
start,
end: start + trimmed.len(),
}
}
#[cfg(test)]
mod tests {
use super::parse;
use crate::{
assert_fails,
error::{Error, Fix},
line_index::LineIndex,
wiki::{Link, Wiki},
};
use std::{
fmt::Write,
path::{Path, PathBuf},
};
fn parse_fixture(source_contents: &str) -> (Wiki, Vec<Error>) {
parse(
Some(Path::new("test.mull")),
source_contents,
&LineIndex::new(source_contents),
)
}
fn parse_test(source_contents: &str) -> Result<Wiki, Vec<Error>> {
let (wiki, errors) = parse_fixture(source_contents);
if errors.is_empty() {
Ok(wiki)
} else {
Err(errors)
}
}
fn link_targets(links: &[Link]) -> Vec<String> {
links
.iter()
.map(|link| match link {
Link::Text { title, .. } => format!("text:{title}"),
Link::Filesystem { target, .. } => format!(
"{}:{}",
if target.is_directory() { "dir" } else { "file" },
target.path().display(),
),
})
.collect()
}
#[test]
fn pages() {
let source =
" \n# Home \n\n Check out the [Greeting]. \n\n# Greeting\n Hello,\nworld! \n";
let wiki = parse_test(source).unwrap();
assert_eq!(wiki.pages.len(), 2);
assert_eq!(wiki.pages["Home"].title, "Home");
assert_eq!(
link_targets(&wiki.pages["Home"].links),
vec!["text:Greeting"],
);
assert_eq!(
wiki.to_string(),
"# Home\n\n Check out the [Greeting].\n\n# Greeting\n\n Hello,\nworld!\n",
);
assert_eq!(wiki.pages["Home"].title_source_range.start, 7);
assert_eq!(wiki.pages["Home"].title_source_range.end, 13);
assert_eq!(wiki.pages["Home"].source_range.start, 3);
assert_eq!(wiki.pages["Home"].source_range.end, 41);
let Link::Text { source_range, .. } = &wiki.pages["Home"].links[0] else {
panic!("The parsed link should be a text link.");
};
assert_eq!(&source[source_range.start..source_range.end], "[Greeting]");
}
#[test]
fn links() {
let wiki = parse_test(concat!(
"# Home\nSee [Greeting], [ About ], and [Greeting].",
"\n# About\n# Greeting",
))
.unwrap();
assert_eq!(
link_targets(&wiki.pages["Home"].links),
vec!["text:Greeting", "text:About", "text:Greeting"],
);
assert_eq!(
wiki.to_string(),
"# Home\n\nSee [Greeting], [About], and [Greeting].\n\n# About\n\n# Greeting\n",
);
}
#[test]
fn unicode_source_ranges() {
let source = "# Home\nSee [Grüße].\n# Grüße";
let wiki = parse_test(source).unwrap();
let Link::Text { source_range, .. } = &wiki.pages["Home"].links[0] else {
panic!("The parsed link should be a text link.");
};
assert_eq!(&source[source_range.start..source_range.end], "[Grüße]");
assert_eq!(
&source[wiki.pages["Grüße"].title_source_range.start
..wiki.pages["Grüße"].title_source_range.end],
"Grüße",
);
}
#[test]
fn escaped_link_delimiters() {
let wiki = parse_test(
r"# Home
See \[Ignored\], [One\]Two], [\[Three], [Four], and \[also ignored\].
# Four
# One]Two
# [Three",
)
.unwrap();
assert_eq!(
link_targets(&wiki.pages["Home"].links),
vec!["text:One]Two", "text:[Three", "text:Four"],
);
}
#[test]
fn escaped_backslashes() {
let wiki = parse_test(
r"# Home
See \\[Four], [Five\\], [A\\B], and \\\[ignored\].
# Four
# Five\
# A\B",
)
.unwrap();
assert_eq!(
link_targets(&wiki.pages["Home"].links),
vec!["text:Four", "text:Five\\", "text:A\\B"],
);
}
#[test]
fn invalid_escapes() {
let (wiki, errors) = parse_fixture("# Home\nC:\\Users \\\\ [Other] \\a\\\nend\\\n# Other");
assert_eq!(
errors.iter().map(Error::fix).collect::<Vec<_>>(),
vec![
Some(&Fix::EscapeBackslash(9)),
Some(&Fix::EscapeBackslash(27)),
Some(&Fix::EscapeBackslash(29)),
Some(&Fix::EscapeBackslash(34)),
],
);
assert!(errors.iter().all(|error| {
error
.to_string()
.contains("A backslash must be followed by")
}));
assert_eq!(link_targets(&wiki.pages["Home"].links), vec!["text:Other"]);
}
#[test]
fn links_with_invalid_escapes() {
let (wiki, errors) = parse_fixture("# Home\n[A\\B] [/c\\d.txt] [Other]\n# Other");
assert_eq!(
errors.iter().map(Error::fix).collect::<Vec<_>>(),
vec![
Some(&Fix::EscapeBackslash(9)),
Some(&Fix::EscapeBackslash(16)),
],
);
assert_eq!(link_targets(&wiki.pages["Home"].links), vec!["text:Other"]);
}
#[test]
fn hashes_cant_be_escaped() {
let (_wiki, errors) = parse_fixture("# Home\nC\\# and \\#1");
assert_eq!(
errors.iter().map(Error::fix).collect::<Vec<_>>(),
vec![
Some(&Fix::EscapeBackslash(8)),
Some(&Fix::EscapeBackslash(15)),
],
);
}
#[test]
fn indented_title_markers() {
let source = "# Home\n\n # Not a title\n";
let wiki = parse_test(source).unwrap();
assert_eq!(wiki.pages.len(), 1);
assert_eq!(wiki.to_string(), source);
}
#[test]
fn filesystem_links() {
let wiki = parse_test(
"# Home\nSee [/notes.txt], [/images/], [/images/./raw/], [/], [//docs/], and \
[/ spaced.txt].",
)
.unwrap();
assert_eq!(
link_targets(&wiki.pages["Home"].links),
vec![
format!("file:{}", PathBuf::from("notes.txt").display()),
format!("dir:{}", PathBuf::from("images").display()),
format!("dir:{}", PathBuf::from("images").join("raw").display()),
"dir:".to_owned(),
format!("dir:{}", PathBuf::from("docs").display()),
format!("file:{}", PathBuf::from(" spaced.txt").display()),
],
);
}
#[test]
fn formatted_links() {
let wiki = parse_test(
"# Home\n[ Home ] [ /a//b/./c\\[1\\].txt ] [/images/] [/images/./raw//] [/] [/./] \
[///]",
)
.unwrap();
assert_eq!(
wiki.to_string(),
"# Home\n\n[Home] [/a/b/c\\[1\\].txt] [/images/] [/images/raw/] [/] [/] [/]\n",
);
}
#[test]
fn trailing_whitespace() {
let wiki =
parse_test("# Home\r\nFirst \t\r\n \nSee [Home] \nand [Home] later.\t\nLast")
.unwrap();
assert_eq!(
wiki.to_string(),
"# Home\n\nFirst\n\nSee [Home]\nand [Home] later.\nLast\n",
);
}
#[test]
fn invalid_filesystem_link_paths() {
let result = parse_test("# Home\n[/../notes.txt] [/notes/../notes.txt]");
assert_fails!(result.clone(), "Path `../notes.txt` must not contain `..`.");
assert_fails!(result, "Path `notes/../notes.txt` must not contain `..`.");
}
#[test]
fn unclosed_link() {
assert_fails!(parse_test("# Home\nSee [Greeting."), "Unclosed link.");
}
#[test]
fn link_with_line_break() {
assert_fails!(
parse_test("# Home\nSee [Greeting\ncontinued]."),
"A link can't contain a line break.",
);
}
#[test]
fn unexpected_opening_delimiter() {
assert_fails!(
parse_test("# Home\nSee [nested[Greeting]."),
"Unexpected opening link delimiter.",
);
}
#[test]
fn unexpected_closing_delimiter() {
let errors = parse_test("# Home\nSee Greeting].").unwrap_err();
assert!(
errors[0]
.to_string()
.contains("Unexpected closing link delimiter"),
);
assert!(errors[0].to_string().contains("2 \u{2502} See Greeting]."));
}
#[test]
fn non_title_hashes() {
let wiki = parse_test("# Home\n\n## Subtitle\n#not a title").unwrap();
assert_eq!(wiki.to_string(), "# Home\n\n## Subtitle\n#not a title\n");
}
#[test]
fn empty_wiki() {
assert!(parse_test(" \n\t\n").unwrap().pages.is_empty());
}
#[test]
fn empty_content() {
assert_eq!(parse_test("# Empty").unwrap().to_string(), "# Empty\n");
}
#[test]
fn windows_line_endings() {
let wiki = parse_test("# Greeting\r\n\r\nHello, world!\r\n").unwrap();
assert_eq!(wiki.to_string(), "# Greeting\n\nHello, world!\n");
}
#[test]
fn content_before_title() {
assert_fails!(
parse_test("Introduction\n# Home"),
"This content isn't in any page.",
);
}
#[test]
fn empty_title() {
let errors = parse_test("# \nContent").unwrap_err();
assert_eq!(errors.len(), 1);
assert!(
errors[0]
.to_string()
.contains("A page title can't be empty."),
);
}
#[test]
fn filesystem_link_titles() {
let errors = parse_test("# Home\n\n# /notes.txt\n\n# /\n\n# notes: /draft").unwrap_err();
assert_eq!(errors.len(), 2);
for error in &errors {
assert!(
error
.to_string()
.contains("A page title can't start with `/`."),
);
}
}
#[test]
fn content_after_empty_title() {
let errors = parse_test("# Home\n\nfoo\n\n#\n\nbar").unwrap_err();
assert_eq!(errors.len(), 1);
assert!(
errors[0]
.to_string()
.contains("A page title can't be empty."),
);
}
#[test]
fn bare_empty_title() {
let errors = parse_test("#").unwrap_err();
assert_eq!(errors.len(), 1);
assert!(
errors[0]
.to_string()
.contains("A page title can't be empty."),
);
assert!(errors[0].to_string().contains("1 │ #"));
}
#[test]
fn repeated_content_before_title() {
let errors = parse_test("First\nSecond\n# Home").unwrap_err();
assert_eq!(errors.len(), 1);
assert!(
errors[0]
.to_string()
.contains("This content isn't in any page."),
);
}
#[test]
fn duplicate_title() {
let errors = parse_test("# Home\nFirst\n# Home\nSecond").unwrap_err();
assert_eq!(errors.len(), 1);
assert!(
errors[0]
.to_string()
.contains("Page `Home` already exists."),
);
assert!(errors[0].to_string().contains("3 \u{2502} # Home"));
}
#[test]
fn multiple_page_errors() {
let errors = parse_test("# First\nUnexpected].\n# Second\nUnclosed [link.").unwrap_err();
assert_eq!(errors.len(), 2);
assert!(
errors[0]
.to_string()
.contains("Unexpected closing link delimiter"),
);
assert!(errors[1].to_string().contains("Unclosed link"));
}
#[test]
fn multiple_errors_in_page() {
let errors = parse_test("# Home\nUnexpected] and [nested[link.").unwrap_err();
assert_eq!(errors.len(), 3);
assert!(
errors[0]
.to_string()
.contains("Unexpected closing link delimiter"),
);
assert!(
errors[1]
.to_string()
.contains("Unexpected opening link delimiter"),
);
assert!(errors[2].to_string().contains("Unclosed link"));
}
#[test]
fn multiple_error_types() {
let errors = parse_test(concat!(
"Introduction\n",
"# \n",
"Content\n",
"# First\n",
"Unexpected].\n",
"# Second\n",
"Unclosed [link.",
))
.unwrap_err();
assert_eq!(errors.len(), 4);
assert!(
errors[0]
.to_string()
.contains("This content isn't in any page"),
);
assert!(
errors[1]
.to_string()
.contains("A page title can't be empty"),
);
assert!(
errors[2]
.to_string()
.contains("Unexpected closing link delimiter"),
);
assert!(errors[3].to_string().contains("Unclosed link"));
}
#[test]
fn recovered_page_keeps_valid_links() {
let (wiki, errors) = parse_fixture(
"# Home\nStray] [Greeting] [Bad\nlink] [nested[link] [/../up] [/notes.txt] [unclosed",
);
assert_eq!(errors.len(), 5);
assert_eq!(
link_targets(&wiki.pages["Home"].links),
vec![
"text:Greeting".to_owned(),
format!("file:{}", PathBuf::from("notes.txt").display()),
],
);
}
#[test]
fn recovered_wiki_has_traversal_order() {
let (wiki, errors) = parse_fixture("# Home\nStray] [Greeting]\n# Greeting");
assert_eq!(errors.len(), 1);
assert_eq!(wiki.pages["Greeting"].traversal_index, Some(1));
}
#[test]
fn recovered_wiki_omits_untitled_regions() {
let (wiki, errors) = parse_fixture(
"Before]\n# Home\n[Home]\n# Home\n[Greeting] a]\n#\n[Greeting] b]\n# Greeting",
);
assert_eq!(errors.len(), 6);
assert!(
errors[0]
.to_string()
.contains("This content isn't in any page."),
);
assert!(errors[1].to_string().contains("1 \u{2502} Before]"));
assert!(
errors[2]
.to_string()
.contains("Page `Home` already exists."),
);
assert!(errors[3].to_string().contains("5 \u{2502} [Greeting] a]"));
assert!(
errors[4]
.to_string()
.contains("A page title can't be empty."),
);
assert!(errors[5].to_string().contains("7 \u{2502} [Greeting] b]"));
assert_eq!(wiki.pages.len(), 2);
assert_eq!(link_targets(&wiki.pages["Home"].links), vec!["text:Home"]);
assert_eq!(wiki.pages["Greeting"].traversal_index, None);
}
}