use crate::{
analyzer::analyze,
cancellation::CancellationFlag,
error::{Error, format_errors},
escaping::Context,
format::CodeStr,
line_index::LineIndex,
parser::{self, backtick_run_length, closing_run_end},
spelled_path::{FileRoot, code_file_path},
wiki::{Link, MIN_FENCE_LENGTH, Page, ScriptBlock, TITLE_PREFIX, Wiki},
};
use sha2::{Digest, Sha256};
use std::{
collections::HashMap,
ffi::OsStr,
fs,
io::{self, Read, Write},
path::{self, Path, PathBuf},
process::{Command, Stdio},
sync::Arc,
thread,
time::{Duration, Instant},
};
pub const SCRIPTS_DIRECTORY: &str = "scripts";
const SCRIPT_TIMEOUT: Duration = Duration::from_secs(30);
pub fn run(
wiki_path: &Path,
source_contents: &str,
rerun: bool,
) -> Result<(String, Vec<ScriptFailure>), Vec<Error>> {
let formatted_contents = analyze(
Some(wiki_path),
source_contents,
&CancellationFlag::default(),
)
.assume_completed()?
.to_string();
run_formatted(wiki_path, &formatted_contents, rerun)
}
pub fn run_formatted(
wiki_path: &Path,
formatted_contents: &str,
rerun: bool,
) -> Result<(String, Vec<ScriptFailure>), Vec<Error>> {
let line_index = LineIndex::new(formatted_contents);
let (wiki, errors) = parser::parse(Some(wiki_path), formatted_contents, &line_index);
if !errors.is_empty() {
return Err(errors);
}
let file_root = FileRoot::new(wiki_path)
.map_err(|message| vec![Error::new(&message, None, None, None, None)])?;
let mut pages = wiki.pages.values().collect::<Vec<_>>();
pages.sort_by_key(|page| page.source_range.start);
let mut replacements = Vec::new();
let mut ran = Vec::new();
let mut failures = Vec::new();
for page in pages {
for (index, block) in page.script_blocks.iter().enumerate() {
if !rerun
&& block.hash.is_some()
&& block.hash == hash(formatted_contents, &wiki, block, &file_root)
{
continue;
}
match run_block(
formatted_contents,
&wiki,
page,
block,
&file_root,
wiki_path,
) {
Ok(replacement) => {
replacements.push((block.source_range, replacement));
ran.push((&page.title, index));
}
Err(failure) => failures.push(ScriptFailure {
error: Error::new(
&failure.message,
Some(wiki_path),
Some((formatted_contents, &line_index, block.name_source_range)),
failure.reason,
None,
),
notification: failure.notification,
}),
}
}
}
let mut run_contents = formatted_contents.to_owned();
for (source_range, replacement) in replacements.into_iter().rev() {
run_contents.replace_range(source_range.start..source_range.end, &replacement);
}
let run_contents = format(wiki_path, &run_contents)?;
let run_line_index = LineIndex::new(&run_contents);
let (run_wiki, errors) = parser::parse(Some(wiki_path), &run_contents, &run_line_index);
if !errors.is_empty() {
return Err(errors);
}
let mut hashes = ran
.into_iter()
.filter_map(|(title, index)| {
let block = &run_wiki.pages.get(title)?.script_blocks[index];
Some((
block.source_range.end,
hash(&run_contents, &run_wiki, block, &file_root)?,
))
})
.collect::<Vec<_>>();
hashes.sort_by_key(|(end, _hash)| *end);
let mut hashed_contents = run_contents.clone();
for (end, hash) in hashes.into_iter().rev() {
hashed_contents.insert_str(end, &format!(" #{hash}"));
}
Ok((format(wiki_path, &hashed_contents)?, failures))
}
fn format(wiki_path: &Path, contents: &str) -> Result<String, Vec<Error>> {
let line_index = LineIndex::new(contents);
let (wiki, errors) = parser::parse(Some(wiki_path), contents, &line_index);
if errors.is_empty() {
Ok(wiki.to_string())
} else {
Err(errors)
}
}
#[cfg(all(test, unix))]
pub fn hide_hashes(contents: &str) -> String {
contents
.split_inclusive('\n')
.map(|line| match line.split_once(" #") {
Some((fence, _hash))
if fence.trim_start().starts_with(':')
&& fence.trim_start().trim_start_matches(':').is_empty() =>
{
format!(
"{fence} #hash{}",
if line.ends_with('\n') { "\n" } else { "" },
)
}
_ => line.to_owned(),
})
.collect()
}
#[cfg(all(test, unix))]
pub fn lock_script_tests() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
LOCK.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn hash(
source_contents: &str,
wiki: &Wiki,
block: &ScriptBlock,
file_root: &FileRoot,
) -> Option<String> {
let script = fs::read(script_path(file_root, &block.name).ok()?).ok()?;
let arguments =
&source_contents[block.arguments_source_range.start..block.arguments_source_range.end];
let body = wiki
.body(block.source_range)
.map(|(body, _line_starts)| body)
.unwrap_or_default();
let mut hasher = Sha256::new();
for part in [script.as_slice(), arguments.as_bytes(), body.as_bytes()] {
hasher.update((part.len() as u64).to_le_bytes());
hasher.update(part);
}
let digest = hasher.finalize();
let mut prefix = [0; 8];
prefix[2..].copy_from_slice(&digest[..6]);
Some(format!("{:012x}", u64::from_be_bytes(prefix)))
}
#[derive(Debug)]
pub struct ScriptFailure {
pub error: Error,
pub notification: Option<String>,
}
struct Failure {
message: String,
reason: Option<Arc<dyn std::error::Error + Send + Sync>>,
notification: Option<String>,
}
impl Failure {
fn new(message: String, reason: Option<Arc<dyn std::error::Error + Send + Sync>>) -> Self {
Self {
message,
reason,
notification: None,
}
}
}
fn run_block(
source_contents: &str,
wiki: &Wiki,
page: &Page,
block: &ScriptBlock,
file_root: &FileRoot,
wiki_path: &Path,
) -> Result<String, Failure> {
let script =
script_path(file_root, &block.name).map_err(|message| Failure::new(message, None))?;
let absolute_script = path::absolute(&script).map_err(|error| {
Failure::new(
"Its path couldn't be found.".to_owned(),
Some(Arc::new(error) as _),
)
})?;
let absolute_file_root = path::absolute(file_root.path()).map_err(|error| {
Failure::new(
"The file root couldn't be found.".to_owned(),
Some(Arc::new(error) as _),
)
})?;
let arguments = arguments(source_contents, page, block, &absolute_file_root);
let input = wiki
.body(block.source_range)
.map(|(body, _line_starts)| body)
.unwrap_or_default();
let output = execute(
&absolute_script,
&arguments,
&input,
&absolute_file_root,
wiki_path,
&page.title.text(Context::Verbatim),
SCRIPT_TIMEOUT,
)
.map_err(|failure| {
let display = script_display(file_root, &script);
Failure {
message: format!("The script {display} failed. {}", failure.message),
reason: failure.reason,
notification: failure.notification.map(|errors| {
if errors.is_empty() {
format!("The script {display} failed.")
} else {
errors
}
}),
}
})?;
let output_source = format!(
"{TITLE_PREFIX}{}\n\n{output}",
page.title.text(Context::Verbatim),
);
let (output_wiki, errors) =
parser::parse(None, &output_source, &LineIndex::new(&output_source));
if !errors.is_empty() || output_wiki.pages.len() != 1 {
return Err(Failure::new(
format!(
"The output of {} can't be read as the content of a page.{}",
script_display(file_root, &script),
if errors.is_empty() {
String::new()
} else {
format!("\n\n{}", format_errors(&errors))
},
),
None,
));
}
let indentation = wiki.indentation(block.source_range.start);
let longest_run = output
.lines()
.map(str::trim_start)
.map(|line| line.len() - line.trim_start_matches(':').len())
.max()
.unwrap_or(0);
let fence = ":".repeat((longest_run + 1).max(MIN_FENCE_LENGTH));
let mut replacement = format!(
"{fence} {}\n",
&source_contents[block.name_source_range.start..block.arguments_source_range.end],
);
for line in output.lines() {
replacement.push_str(indentation);
replacement.push_str(line);
replacement.push('\n');
}
replacement.push_str(indentation);
replacement.push_str(&fence);
Ok(replacement)
}
pub fn script_path(file_root: &FileRoot, name: &str) -> Result<PathBuf, String> {
let directory = file_root.path().join(SCRIPTS_DIRECTORY);
let mut paths = fs::read_dir(&directory)
.into_iter()
.flatten()
.flatten()
.map(|entry| entry.path())
.filter(|path| path.is_file() && path.file_stem() == Some(OsStr::new(name)))
.collect::<Vec<_>>();
paths.sort();
match paths.as_slice() {
[path] => Ok(path.clone()),
[] => Err(format!(
"No script named {} in {}.",
name.code_str(),
code_file_path(Path::new(SCRIPTS_DIRECTORY)),
)),
_ => Err(format!(
"More than one script is named {} in {}: {}.",
name.code_str(),
code_file_path(Path::new(SCRIPTS_DIRECTORY)),
paths
.iter()
.map(|path| script_display(file_root, path))
.collect::<Vec<_>>()
.join(", "),
)),
}
}
pub fn script_names(file_root: &FileRoot) -> Vec<String> {
let mut names = fs::read_dir(file_root.path().join(SCRIPTS_DIRECTORY))
.into_iter()
.flatten()
.flatten()
.map(|entry| entry.path())
.filter(|path| path.is_file())
.filter_map(|path| Some(path.file_stem()?.to_str()?.to_owned()))
.collect::<Vec<_>>();
names.sort();
names.dedup();
names
}
fn script_display(file_root: &FileRoot, script: &Path) -> String {
code_file_path(script.strip_prefix(file_root.path()).unwrap_or(script)).to_string()
}
fn arguments(
source_contents: &str,
page: &Page,
block: &ScriptBlock,
file_root: &Path,
) -> Vec<String> {
let source_range = block.arguments_source_range;
let links = page
.links
.iter()
.filter(|link| {
source_range.start <= link.source_range().start
&& link.source_range().end <= source_range.end
})
.map(|link| (link.source_range().start, link))
.collect::<HashMap<_, _>>();
let mut arguments = Vec::new();
let mut argument = None::<String>;
let mut position = source_range.start;
while let Some(character) = source_contents[position..source_range.end].chars().next() {
if let Some(link) = links.get(&position) {
let value = match link {
Link::Text { title, .. } => title.text(Context::Verbatim),
Link::Filesystem { target, .. } => {
file_root.join(target.path()).to_string_lossy().into_owned()
}
Link::Malformed { source_range, .. } => {
source_contents[source_range.start..source_range.end].to_owned()
}
};
argument.get_or_insert_default().push_str(&value);
position = link.source_range().end;
continue;
}
let rest = &source_contents[position..source_range.end];
if character.is_whitespace() {
arguments.extend(argument.take());
position += character.len_utf8();
} else if character == '`' {
let length = backtick_run_length(rest);
let code_start = position + length;
let (code_end, code_run_end) = closing_run_end(source_contents, code_start, length)
.filter(|&run_end| run_end <= source_range.end)
.map_or((source_range.end, source_range.end), |run_end| {
(run_end - length, run_end)
});
argument
.get_or_insert_default()
.push_str(&source_contents[code_start..code_end]);
position = code_run_end;
} else if character == '\\' {
let escaped = rest[1..].chars().next().unwrap_or('\\');
argument.get_or_insert_default().push(escaped);
position += '\\'.len_utf8() + escaped.len_utf8();
} else {
argument.get_or_insert_default().push(character);
position += character.len_utf8();
}
}
arguments.extend(argument);
arguments
}
fn execute(
script: &Path,
arguments: &[String],
input: &str,
file_root: &Path,
wiki_path: &Path,
title: &str,
timeout: Duration,
) -> Result<String, Failure> {
let mut command = if cfg!(windows) {
let mut command = Command::new("cmd");
command.arg("/C").arg(script);
command
} else {
Command::new(script)
};
let mut child = command
.args(arguments)
.current_dir(file_root)
.env(
"MULL_WIKI",
path::absolute(wiki_path).unwrap_or_else(|_| wiki_path.to_owned()),
)
.env("MULL_PAGE", title)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|error| {
let message = if error.kind() == io::ErrorKind::PermissionDenied {
"It isn't executable.".to_owned()
} else {
"It couldn't be started.".to_owned()
};
Failure::new(message, Some(Arc::new(error) as _))
})?;
let mut stdin = child
.stdin
.take()
.expect("The script's input should be piped.");
let input = input.to_owned();
let writer = thread::spawn(move || stdin.write_all(input.as_bytes()));
let stdout_reader = read_in_background(
child
.stdout
.take()
.expect("The script's output should be piped."),
);
let stderr_reader = read_in_background(
child
.stderr
.take()
.expect("The script's errors should be piped."),
);
let start = Instant::now();
let status = loop {
if let Some(status) = child.try_wait().map_err(|error| {
Failure::new(
"It couldn't be awaited.".to_owned(),
Some(Arc::new(error) as _),
)
})? {
break status;
}
if start.elapsed() > timeout {
let _ = child.kill();
let _ = child.wait();
return Err(Failure::new(
format!("It ran for longer than {} seconds.", timeout.as_secs_f64()),
None,
));
}
thread::sleep(Duration::from_millis(10));
};
let _ = writer.join();
let stdout = stdout_reader.join().unwrap_or_default();
let stderr = stderr_reader.join().unwrap_or_default();
let stderr = String::from_utf8_lossy(&stderr).trim().to_owned();
let with_errors = |message: String| {
if stderr.is_empty() {
message
} else {
format!("{message}\n\n{stderr}")
}
};
if !status.success() {
let message = status.code().map_or_else(
|| "It was stopped by a signal.".to_owned(),
|code| format!("It exited with status {code}."),
);
return Err(Failure {
message: with_errors(message),
reason: None,
notification: Some(stderr.clone()),
});
}
String::from_utf8(stdout).map_err(|_| {
Failure::new(
with_errors("Its output isn't valid UTF-8.".to_owned()),
None,
)
})
}
fn read_in_background<R: Read + Send + 'static>(mut reader: R) -> thread::JoinHandle<Vec<u8>> {
thread::spawn(move || {
let mut output = Vec::new();
let _ = reader.read_to_end(&mut output);
output
})
}
#[cfg(test)]
mod tests {
use super::arguments;
use crate::{line_index::LineIndex, parser};
use std::path::Path;
#[cfg(unix)]
use {
super::{ScriptFailure, execute, hide_hashes, lock_script_tests, run},
crate::error::Error,
std::{
env, fs,
os::unix::fs::PermissionsExt,
path::PathBuf,
process,
sync::atomic::{AtomicUsize, Ordering},
time::Duration,
},
};
#[cfg(unix)]
static NEXT_DIRECTORY: AtomicUsize = AtomicUsize::new(0);
#[cfg(unix)]
struct TestWiki(PathBuf);
#[cfg(unix)]
impl TestWiki {
fn new(source: &str, scripts: &[(&str, &str)]) -> Self {
let directory = env::temp_dir().join(format!(
"mull-scripts-{}-{}",
process::id(),
NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed),
));
fs::create_dir_all(directory.join("wiki_files/scripts")).unwrap();
fs::write(directory.join("wiki.mull"), source).unwrap();
for (name, contents) in scripts {
let path = directory.join("wiki_files/scripts").join(name);
fs::write(&path, contents).unwrap();
fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap();
}
Self(directory)
}
fn path(&self) -> PathBuf {
self.0.join("wiki.mull")
}
fn run(&self) -> Result<(String, Vec<ScriptFailure>), Vec<Error>> {
run(
&self.path(),
&fs::read_to_string(self.path()).unwrap(),
false,
)
.map(|(contents, failures)| (hide_hashes(&contents), failures))
}
fn run_and_save(&self, rerun: bool) -> usize {
let (contents, failures) = run(
&self.path(),
&fs::read_to_string(self.path()).unwrap(),
rerun,
)
.unwrap();
assert!(failures.is_empty(), "{failures:?}");
fs::write(self.path(), contents).unwrap();
fs::read_to_string(self.0.join("wiki_files/runs"))
.unwrap_or_default()
.lines()
.count()
}
}
#[cfg(unix)]
impl Drop for TestWiki {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
#[cfg(unix)]
#[test]
fn runs_replace_bodies() {
let _lock = lock_script_tests();
let wiki = TestWiki::new(
concat!(
"# Home\n\n[/scripts/] [Other]\n\n",
"- List:\n ::: sort\n - b\n - a\n :::\n\n",
"::: fence :::\n\n",
"# Other\n",
),
&[
("sort.sh", "#!/bin/sh\nsort\n"),
(
"fence.sh",
"#!/bin/sh\necho \"$MULL_PAGE\"\necho '::: [Other] :::'\n",
),
],
);
let (contents, failures) = wiki.run().unwrap();
assert!(failures.is_empty(), "{failures:?}");
assert_eq!(
contents,
concat!(
"# Home\n\n[/scripts/] [Other]\n\n",
"- List:\n ::: sort\n - a\n - b\n ::: #hash\n\n",
":::: fence\nHome\n::: [Other] :::\n:::: #hash\n\n",
"# Other\n",
),
);
}
#[cfg(unix)]
#[test]
fn failed_runs() {
let _lock = lock_script_tests();
let wiki = TestWiki::new(
concat!(
"# Home\n\n[/scripts/]\n\n",
"::: fail :::\n\n::: bracket :::\n\n::: plain :::\n\n::: quiet :::\n",
),
&[
("fail.sh", "#!/bin/sh\necho nope >&2\nexit 3\n"),
("quiet.sh", "#!/bin/sh\nexit 1\n"),
("bracket.sh", "#!/bin/sh\necho ]\n"),
("plain.txt", "Not a script."),
],
);
let plain_script = wiki.0.join("wiki_files/scripts/plain.txt");
fs::set_permissions(&plain_script, fs::Permissions::from_mode(0o644)).unwrap();
let (contents, failures) = wiki.run().unwrap();
assert_eq!(contents, fs::read_to_string(wiki.path()).unwrap());
let messages = failures
.iter()
.map(|failure| failure.error.to_string())
.collect::<Vec<_>>();
assert_eq!(messages.len(), 4);
assert!(
messages[0]
.contains("The script `/scripts/fail.sh` failed. It exited with status 3.\n\nnope"),
);
assert!(messages[1].contains(
"The output of `/scripts/bracket.sh` can't be read as the content of a page.",
));
assert!(
messages[2].contains("The script `/scripts/plain.txt` failed. It isn't executable."),
);
assert!(
messages[3].contains("The script `/scripts/quiet.sh` failed. It exited with status 1."),
);
assert_eq!(
failures
.iter()
.map(|failure| failure.notification.as_deref())
.collect::<Vec<_>>(),
vec![
Some("nope"),
None,
None,
Some("The script `/scripts/quiet.sh` failed."),
],
);
}
#[cfg(unix)]
#[test]
fn runs_skip_unchanged_blocks() {
let _lock = lock_script_tests();
let wiki = TestWiki::new(
"# Home\n\n[/runs] [/scripts/]\n\n::: count a\n- b\n:::\n",
&[("count.sh", "#!/bin/sh\necho run >> runs\ncat\n")],
);
fs::write(wiki.0.join("wiki_files/runs"), "").unwrap();
assert_eq!(wiki.run_and_save(false), 1);
assert_eq!(wiki.run_and_save(false), 1);
assert_eq!(wiki.run_and_save(true), 2);
let source = fs::read_to_string(wiki.path()).unwrap();
fs::write(wiki.path(), source.replace("- b", "- c")).unwrap();
assert_eq!(wiki.run_and_save(false), 3);
let source = fs::read_to_string(wiki.path()).unwrap();
fs::write(wiki.path(), source.replace("count a", "count b")).unwrap();
assert_eq!(wiki.run_and_save(false), 4);
let script = wiki.0.join("wiki_files/scripts/count.sh");
fs::write(&script, "#!/bin/sh\necho run >> runs\ncat\ntrue\n").unwrap();
assert_eq!(wiki.run_and_save(false), 5);
assert_eq!(wiki.run_and_save(false), 5);
}
#[test]
fn script_arguments() {
let source =
"# Home\n::: tidy [Home] [/notes.txt] `a\\d b` x\\]y two words --page=[Home] :::";
let (wiki, errors) = parser::parse(None, source, &LineIndex::new(source));
assert!(errors.is_empty());
let page = wiki.page("Home");
assert_eq!(
arguments(source, page, &page.script_blocks[0], Path::new("/root")),
vec![
"Home",
&Path::new("/root").join("notes.txt").to_string_lossy(),
"a\\d b",
"x]y",
"two",
"words",
"--page=Home",
],
);
}
#[cfg(unix)]
#[test]
fn runs_time_out() {
let _lock = lock_script_tests();
let wiki = TestWiki::new("# Home\n", &[("slow.sh", "#!/bin/sh\nsleep 5\n")]);
let failure = execute(
&wiki.0.join("wiki_files/scripts/slow.sh"),
&[],
"",
&wiki.0.join("wiki_files"),
&wiki.path(),
"Home",
Duration::from_millis(100),
)
.unwrap_err();
assert_eq!(failure.message, "It ran for longer than 0.1 seconds.");
}
}