use std::collections::HashMap;
use std::ops::Range;
use std::path::{Path, PathBuf};
use bynk_emit::project::AttributedError;
use bynk_syntax::ast::SourceUnit;
use bynk_syntax::lexer::{TokenKind, doc_block_body_range, tokenize};
use bynk_syntax::parser::parse_unit_with_recovery;
const SYNTHETIC_SUFFIX: &str = ".doc-examples.bynk";
const ITEMS_INFIX: &str = "_doc_example_";
#[derive(Debug, PartialEq, Eq)]
enum Fence {
Foreign,
Skipped,
Checked,
Run,
}
fn classify(info: &str) -> Fence {
let mut words = info
.split(|c: char| c == ',' || c.is_whitespace())
.filter(|w| !w.is_empty());
if words.next() != Some("bynk") {
return Fence::Foreign;
}
let words: Vec<&str> = words.collect();
if words
.iter()
.any(|w| matches!(*w, "ignore" | "fragment" | "fail") || w.starts_with("fail="))
{
Fence::Skipped
} else if words.contains(&"test") {
Fence::Run
} else {
Fence::Checked
}
}
struct DocLine<'a> {
text: &'a str,
start: usize,
}
fn doc_blocks(source: &str) -> Vec<Vec<DocLine<'_>>> {
let Ok(tokens) = tokenize(source) else {
return Vec::new();
};
let mut blocks = Vec::new();
for t in &tokens {
if t.kind != TokenKind::DocBlock {
continue;
}
let Some(range) = doc_block_body_range(source, t.span) else {
continue;
};
let mut lines = Vec::new();
let mut at = range.start;
for raw in source[range.clone()].split('\n') {
lines.push(DocLine {
text: raw.trim_end_matches('\r'),
start: at,
});
at += raw.len() + 1;
}
blocks.push(lines);
}
blocks
}
struct Example<'a> {
run: bool,
open: usize,
body: Vec<DocLine<'a>>,
}
impl Example<'_> {
fn close(&self) -> usize {
self.open + 1 + self.body.len()
}
fn text(&self) -> String {
self.body
.iter()
.map(|l| l.text)
.collect::<Vec<_>>()
.join("\n")
}
}
fn fence_marker(line: &str) -> Option<(usize, &str)> {
let trimmed = line.trim_start();
let ticks = trimmed.bytes().take_while(|&b| b == b'`').count();
(ticks >= 3).then(|| (ticks, &trimmed[ticks..]))
}
fn examples(source: &str) -> Vec<Example<'_>> {
let line_of = |offset: usize| source[..offset].matches('\n').count();
let mut out = Vec::new();
for block in doc_blocks(source) {
let mut open: Option<(usize, Fence, usize, Vec<DocLine<'_>>)> = None;
for line in block {
match (fence_marker(line.text), open.as_mut()) {
(None, Some((_, _, _, body))) => body.push(line),
(None, None) => {}
(Some((ticks, info)), None) => {
open = Some((ticks, classify(info), line_of(line.start), Vec::new()));
}
(Some((ticks, info)), Some((n, ..))) if ticks >= *n && info.trim().is_empty() => {
let (_, kind, open_line, body) = open.take().expect("matched Some");
if matches!(kind, Fence::Checked | Fence::Run) {
out.push(Example {
run: kind == Fence::Run,
open: open_line,
body,
});
}
}
(Some(_), Some((_, _, _, body))) => body.push(line),
}
}
if let Some((_, kind @ (Fence::Checked | Fence::Run), open_line, body)) = open {
out.push(Example {
run: kind == Fence::Run,
open: open_line,
body,
});
}
}
out
}
#[derive(Debug, PartialEq, Eq)]
enum Body {
Statements,
Items,
}
fn parses_clean(text: &str, wanted: impl Fn(&SourceUnit) -> bool) -> bool {
let Ok(tokens) = tokenize(text) else {
return false;
};
let (unit, errors) = parse_unit_with_recovery(&tokens, text);
errors.is_empty() && unit.as_ref().is_some_and(wanted)
}
fn classify_body(target: &str, body: &str) -> Body {
let case = format!("suite {target} {{\n case \"x\" {{\n{body}\n }}\n}}\n");
if parses_clean(&case, |u| matches!(u, SourceUnit::Suite(_))) {
return Body::Statements;
}
let items = format!("commons {target}{ITEMS_INFIX}0\n\n{body}\n");
if parses_clean(&items, |u| matches!(u, SourceUnit::Commons(_))) {
Body::Items
} else {
Body::Statements
}
}
fn line_starts(text: &str) -> Vec<usize> {
std::iter::once(0)
.chain(text.match_indices('\n').map(|(i, _)| i + 1))
.collect()
}
struct Table {
synth_key: PathBuf,
synth_starts: Vec<usize>,
real_starts: Vec<usize>,
real_len: usize,
}
impl Table {
fn locate(&self, offset: usize) -> (usize, Range<usize>) {
let line = self
.synth_starts
.partition_point(|&s| s <= offset)
.saturating_sub(1);
let start = self.real_starts[line];
let end = self
.real_starts
.get(line + 1)
.map_or(self.real_len, |next| next - 1);
(offset - self.synth_starts[line], start..end)
}
fn map_range(&self, start: usize, end: usize) -> (usize, usize) {
let (col, line) = self.locate(start);
if line.start + col > line.end {
return (line.start, line.end);
}
let from = line.start + col;
let (end_col, end_line) = self.locate(end.max(start));
(from, (end_line.start + end_col).min(end_line.end).max(from))
}
}
fn suite_text(
target: &str,
file: &str,
n_lines: usize,
examples: &[&Example<'_>],
) -> Option<String> {
let mut lines = vec![String::new(); n_lines];
for (i, ex) in examples.iter().enumerate() {
let head = if i == 0 {
format!("suite {target} {{ ")
} else {
String::new()
};
lines[ex.open] = format!("{head}case \"doc example {file}:{}\" {{", ex.open + 1);
for (k, l) in ex.body.iter().enumerate() {
lines[ex.open + 1 + k] = l.text.to_string();
}
let tail = if i + 1 == examples.len() { "} }" } else { "}" };
lines[ex.close()] = tail.to_string();
}
(!examples.is_empty()).then(|| lines.join("\n"))
}
fn items_text(target: &str, n: usize, n_lines: usize, ex: &Example<'_>) -> String {
let mut lines = vec![String::new(); n_lines];
lines[0] = format!("commons {target}{ITEMS_INFIX}{n}");
lines[ex.open] = format!("uses {target}");
for (k, l) in ex.body.iter().enumerate() {
lines[ex.open + 1 + k] = l.text.to_string();
}
lines.join("\n")
}
fn real_name_of(synthetic: &str) -> Option<String> {
if let Some(stem) = synthetic.strip_suffix(SYNTHETIC_SUFFIX) {
return Some(format!("{stem}.bynk"));
}
let stem = synthetic.strip_suffix(".bynk")?;
let (real, n) = stem.rsplit_once(ITEMS_INFIX)?;
(!n.is_empty() && n.chars().all(|c| c.is_ascii_digit())).then(|| format!("{real}.bynk"))
}
#[derive(Default)]
pub struct DocExamples {
check: Vec<(PathBuf, String)>,
run: Vec<(PathBuf, String)>,
tables: Vec<Table>,
count: usize,
}
impl DocExamples {
pub fn collect(sources: &HashMap<PathBuf, String>) -> Self {
let mut keys: Vec<&PathBuf> = sources.keys().collect();
keys.sort();
let mut found = DocExamples::default();
for key in keys {
let source = &sources[key];
let Ok(tokens) = tokenize(source) else {
continue;
};
let (unit, errors) = parse_unit_with_recovery(&tokens, source);
let (target, is_commons) = match unit {
Some(SourceUnit::Commons(c)) if errors.is_empty() => (c.name.joined(), true),
Some(SourceUnit::Context(c)) if errors.is_empty() => (c.name.joined(), false),
_ => continue,
};
let examples = examples(source);
if examples.is_empty() {
continue;
}
found.add_unit(sources, key, &target, is_commons, &examples);
}
found
}
fn add_unit(
&mut self,
sources: &HashMap<PathBuf, String>,
key: &Path,
target: &str,
is_commons: bool,
examples: &[Example<'_>],
) {
let source = &sources[key];
let file = key.file_name().unwrap_or_default().to_string_lossy();
let stem = key.file_stem().unwrap_or_default().to_string_lossy();
let real_starts = line_starts(source);
let n_lines = real_starts.len();
let table = |synth_key: &PathBuf, text: &str| Table {
synth_key: synth_key.clone(),
synth_starts: line_starts(text),
real_starts: real_starts.clone(),
real_len: source.len(),
};
let mut cases: Vec<&Example<'_>> = Vec::new();
let mut items_n = 0;
for ex in examples {
match classify_body(target, &ex.text()) {
Body::Statements => {
cases.push(ex);
self.count += 1;
}
Body::Items if !is_commons => {}
Body::Items => {
let synth_key = loop {
items_n += 1;
let k = key.with_file_name(format!("{stem}{ITEMS_INFIX}{items_n}.bynk"));
if !sources.contains_key(&k) {
break k;
}
};
self.count += 1;
let text = items_text(target, items_n, n_lines, ex);
self.tables.push(table(&synth_key, &text));
self.check.push((synth_key, text));
}
}
}
let synth_key = key.with_file_name(format!("{stem}{SYNTHETIC_SUFFIX}"));
if sources.contains_key(&synth_key) {
self.count -= cases.len();
return;
}
if let Some(text) = suite_text(target, &file, n_lines, &cases) {
self.tables.push(table(&synth_key, &text));
self.check.push((synth_key.clone(), text));
}
let runnable: Vec<&Example<'_>> = cases.iter().copied().filter(|e| e.run).collect();
if let Some(text) = suite_text(target, &file, n_lines, &runnable) {
self.run.push((synth_key, text));
}
}
pub fn is_empty(&self) -> bool {
self.check.is_empty()
}
pub fn example_count(&self) -> usize {
self.count
}
pub fn has_runnable(&self) -> bool {
!self.run.is_empty()
}
pub fn check_overlay(&self, sources: &HashMap<PathBuf, String>) -> HashMap<PathBuf, String> {
let mut out = sources.clone();
out.extend(self.check.iter().cloned());
out
}
pub fn run_overlay(&self, sources: &HashMap<PathBuf, String>) -> HashMap<PathBuf, String> {
let mut out = sources.clone();
out.extend(self.run.iter().cloned());
out
}
pub fn run_aliases(&self) -> Vec<(String, String)> {
self.run
.iter()
.filter_map(|(key, _)| {
let name = key.file_name()?.to_string_lossy().into_owned();
let real = real_name_of(&name)?;
Some((name, real))
})
.collect()
}
pub fn remap(&self, errors: Vec<AttributedError>) -> Vec<AttributedError> {
errors
.into_iter()
.filter(|ae| {
bynk_syntax::Severity::for_error(&ae.error) == bynk_syntax::Severity::Error
})
.map(|ae| match self.table_for(&ae) {
Some((table, real)) => Self::remap_one(ae, table, real),
None => {
let mut ae = ae;
ae.error.notes.push(
"this appeared when the doc-comment examples were checked".to_string(),
);
ae
}
})
.collect()
}
fn table_for(&self, ae: &AttributedError) -> Option<(&Table, PathBuf)> {
let path = ae.source_path.as_deref()?;
let name = path.file_name()?.to_string_lossy().into_owned();
let real = path.with_file_name(real_name_of(&name)?);
let table = self.tables.iter().find(|t| t.synth_key.ends_with(path))?;
Some((table, real))
}
fn remap_one(mut ae: AttributedError, table: &Table, real: PathBuf) -> AttributedError {
let map = |span: bynk_syntax::span::Span| {
let (start, end) = table.map_range(span.start, span.end);
bynk_syntax::span::Span { start, end, ..span }
};
ae.error.span = map(ae.error.span);
for (i, (span, _)) in ae.error.labels.iter_mut().enumerate() {
if matches!(
ae.label_files.get(i),
None | Some(bynk_emit::project::LabelFile::Primary)
) {
*span = map(*span);
}
}
ae.error.suggestions.clear();
ae.source_path = Some(real);
ae
}
}
fn is_name_char(c: char) -> bool {
c.is_alphanumeric() || matches!(c, '_' | '-' | '.')
}
fn name_occurrences<'t>(text: &'t str, name: &'t str) -> impl Iterator<Item = usize> + 't {
text.match_indices(name).map(|(i, _)| i).filter(move |&i| {
let before = text[..i].chars().next_back();
let after = text[i + name.len()..].chars().next();
!before.is_some_and(is_name_char) && !after.is_some_and(is_name_char)
})
}
pub fn is_synthetic_path(path: &str, aliases: &[(String, String)]) -> bool {
aliases.iter().any(|(synthetic, _)| {
path.ends_with(synthetic.as_str()) && name_occurrences(path, synthetic).count() > 0
})
}
pub fn apply_aliases<'a>(text: &'a str, aliases: &[(String, String)]) -> std::borrow::Cow<'a, str> {
let mut out = std::borrow::Cow::Borrowed(text);
for (synthetic, real) in aliases {
let at: Vec<usize> = name_occurrences(&out, synthetic).collect();
if at.is_empty() {
continue;
}
let mut replaced = String::with_capacity(out.len());
let mut last = 0;
for i in at {
replaced.push_str(&out[last..i]);
replaced.push_str(real);
last = i + synthetic.len();
}
replaced.push_str(&out[last..]);
out = std::borrow::Cow::Owned(replaced);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn classifies_info_strings() {
assert_eq!(classify("bynk"), Fence::Checked);
assert_eq!(classify(" bynk "), Fence::Checked);
assert_eq!(classify("bynk test"), Fence::Run);
assert_eq!(classify("bynk,test"), Fence::Run);
assert_eq!(classify("bynk run"), Fence::Checked);
assert_eq!(classify("bynk ignore"), Fence::Skipped);
assert_eq!(classify("bynk,ignore"), Fence::Skipped);
assert_eq!(classify("bynk test ignore"), Fence::Skipped);
assert_eq!(classify("bynk,fragment"), Fence::Skipped);
assert_eq!(classify("bynk,fail=bynk.types.mismatch"), Fence::Skipped);
assert_eq!(classify("ts"), Fence::Foreign);
assert_eq!(classify(""), Fence::Foreign);
assert_eq!(classify("bynkc"), Fence::Foreign);
}
fn sources(text: &str) -> HashMap<PathBuf, String> {
HashMap::from([(PathBuf::from("src/m.bynk"), text.to_string())])
}
#[test]
fn finds_doc_block_examples() {
let src = "---\nSums.\n\n```bynk\nexpect add(1, 2) == 3\n```\n\n```bynk ignore\nnope\n```\n---\ncommons m\n\n---\n```bynk\nexpect true\n```\n---\nfn add(a: Int, b: Int) -> Int { a + b }\n";
let found = DocExamples::collect(&sources(src));
assert_eq!(found.example_count(), 2);
let (_, text) = &found.check[0];
assert!(text.starts_with("\n\n\nsuite m { case "), "{text}");
assert!(text.contains("expect add(1, 2) == 3\n"), "{text}");
assert!(text.contains("expect true\n"), "{text}");
assert!(!text.contains("nope"), "{text}");
assert!(!found.has_runnable());
}
#[test]
fn a_synthetic_file_keeps_every_line_of_the_real_one() {
let src =
"---\nProse.\n\n```bynk test\n let x = 1\n\texpect x == 1\n```\n---\ncommons m\n";
let found = DocExamples::collect(&sources(src));
assert!(found.has_runnable());
let (_, text) = &found.run[0];
let want: Vec<&str> = src.split('\n').collect();
let got: Vec<&str> = text.split('\n').collect();
assert_eq!(got.len(), want.len(), "{text}");
assert_eq!(got[4], want[4]);
assert_eq!(got[5], want[5]);
assert!(got[3].starts_with("suite m { case \"doc example m.bynk:4\" {"));
assert_eq!(got[6], "} }");
}
#[test]
fn maps_a_body_offset_back_to_the_fence() {
let src = "---\n```bynk\nexpect 1 == 1\n```\n---\ncommons m\n";
let found = DocExamples::collect(&sources(src));
let (_, text) = &found.check[0];
let synth = text.find("expect").unwrap();
let table = &found.tables[0];
let (start, end) = table.map_range(synth + 7, synth + 8);
assert_eq!(&src[start..end], "1");
let (hs, he) = table.map_range(text.find("case").unwrap(), text.find("case").unwrap() + 4);
assert!(src[hs..he].starts_with("```"), "{:?}", &src[hs..he]);
}
#[test]
fn classifies_bodies_by_parsing_them() {
assert_eq!(classify_body("m", "expect 1 == 1"), Body::Statements);
assert_eq!(
classify_body("m", "let x = 1\nexpect x == 1"),
Body::Statements
);
assert_eq!(
classify_body("m", "type P = { a: Int }\n\nfn f(p: P) -> Int { p.a }"),
Body::Items
);
assert_eq!(classify_body("m", "this is nonsense ("), Body::Statements);
}
#[test]
fn a_declaration_fence_is_a_sibling_commons_that_uses_a_commons() {
let src = "---\n```bynk\nfn f() -> Int { 1 }\n```\n---\ncommons m\n";
let found = DocExamples::collect(&sources(src));
assert_eq!(found.check.len(), 1);
let (key, text) = &found.check[0];
assert!(key.ends_with("m_doc_example_1.bynk"), "{key:?}");
assert!(text.starts_with("commons m_doc_example_1\n"), "{text}");
assert!(text.contains("\nuses m\n"), "{text}");
assert_eq!(
real_name_of("m_doc_example_1.bynk").as_deref(),
Some("m.bynk")
);
assert_eq!(
real_name_of("m.doc-examples.bynk").as_deref(),
Some("m.bynk")
);
assert_eq!(real_name_of("m.bynk"), None);
assert_eq!(real_name_of("m_doc_example_x.bynk"), None);
}
#[test]
fn a_declaration_fence_in_a_context_is_not_checked() {
let src = "---\n```bynk\nfn f(o: Order) -> Int { 1 }\n```\n---\ncontext m\n\ntype Order = { c: Int }\n";
let found = DocExamples::collect(&sources(src));
assert!(found.is_empty());
assert_eq!(found.example_count(), 0);
}
#[test]
fn aliases_show_the_real_file_name() {
let aliases = vec![("m.doc-examples.bynk".to_string(), "m.bynk".to_string())];
assert_eq!(
apply_aliases("at src/m.doc-examples.bynk:5:3", &aliases),
"at src/m.bynk:5:3"
);
assert!(matches!(
apply_aliases("nothing here", &aliases),
std::borrow::Cow::Borrowed(_)
));
}
#[test]
fn a_longer_fence_keeps_a_shorter_one_shown_inside_it() {
let src = "---\n````text\n```bynk\nnope nope\n```\n````\n---\ncommons m\n";
assert!(DocExamples::collect(&sources(src)).is_empty());
}
#[test]
fn an_unterminated_fence_is_checked_to_the_end_of_the_doc() {
let src = "---\n```bynk\nexpect 1 == 1\n---\ncommons m\n";
let found = DocExamples::collect(&sources(src));
assert_eq!(found.example_count(), 1);
let (_, text) = &found.check[0];
assert!(text.contains("expect 1 == 1\n"), "{text}");
}
#[test]
fn a_real_file_named_like_a_synthetic_one_is_never_replaced() {
let src = "---\n```bynk\nexpect 1 == 1\n```\n---\ncommons m\n";
let mut all = sources(src);
all.insert(PathBuf::from("src/m.doc-examples.bynk"), "x".to_string());
let found = DocExamples::collect(&all);
assert!(
found.is_empty(),
"the suite is left unchecked, not written over"
);
assert_eq!(found.example_count(), 0);
let items = "---\n```bynk\nfn f() -> Int { 1 }\n```\n---\ncommons m\n";
let mut all = sources(items);
all.insert(PathBuf::from("src/m_doc_example_1.bynk"), "x".to_string());
let found = DocExamples::collect(&all);
assert!(found.check[0].0.ends_with("m_doc_example_2.bynk"));
}
#[test]
fn aliases_only_touch_a_whole_file_name() {
let aliases = vec![(
"calc.doc-examples.bynk".to_string(),
"calc.bynk".to_string(),
)];
assert_eq!(
apply_aliases("at src/demo/calc.doc-examples.bynk:5:3", &aliases),
"at src/demo/calc.bynk:5:3"
);
assert_eq!(
apply_aliases("src/xcalc.doc-examples.bynk", &aliases),
"src/xcalc.doc-examples.bynk"
);
assert_eq!(
apply_aliases("calc.doc-examples.bynk.map", &aliases),
"calc.doc-examples.bynk.map"
);
assert!(is_synthetic_path(
"src/demo/calc.doc-examples.bynk",
&aliases
));
assert!(is_synthetic_path("calc.doc-examples.bynk", &aliases));
assert!(!is_synthetic_path("src/xcalc.doc-examples.bynk", &aliases));
}
}