blue_lang_syntax/
doc.rs

1//! `@doc` and `@moduledoc`: documentation as a typed child of the tree.
2//!
3//! ```text
4//! @moduledoc """
5//! retsu holds the list verbs.
6//!
7//!     retsu::first([1, 2]) #=> 1
8//! """
9//!
10//! @doc """
11//! clamp keeps a number inside a range.
12//!
13//!     clamp(99, 1, 10) #=> 10
14//! """
15//! def clamp(x, lo, hi)
16//!   …
17//! end
18//! ```
19//!
20//! `@doc` lowers to `(blue-doc "text" (define …))`: the definition is a child
21//! of its documentation, so the text travels with the tree it describes, into
22//! the formatter, `def_hash`, the doc rules and mondou's views. `@moduledoc` is
23//! the first form of a file and lowers to `(blue-moduledoc "text")`.
24//!
25//! **Two views of one parse.** [`crate::parse_program_documented`] returns the
26//! tree with these nodes. Every other `parse_program*` door returns
27//! [`strip`]ped forms: a documented definition becomes the definition itself,
28//! spanning its documentation too, and the module doc is gone. So the checker,
29//! the resolver and the evaluators see exactly the tree they saw before docs
30//! existed, and only the consumers that read docs ask for them.
31//!
32//! The text is data, not code: no escapes and no interpolation. Its
33//! indentation (the closing `"""`'s), trailing whitespace and leading and
34//! trailing blank lines are stripped at lex time, so the formatter writes one
35//! spelling of a text.
36//!
37//! **Examples are doctests.** An indented paragraph of the text that holds a
38//! line `expr #=> value` is an example: its other lines run first, and each
39//! `#=>` line asserts `expr == value`. [`doctests`] lowers each example to a
40//! `test` block, so `blue test` runs it on the evaluator that runs every other
41//! test, and a red example is a red build.
42
43use tatara_lisp::{Atom, Span, Spanned, SpannedForm};
44
45use crate::ParseError;
46
47/// The head `@doc` lowers to.
48pub const DOC_HEAD: &str = "blue-doc";
49
50/// The head `@moduledoc` lowers to.
51pub const MODULEDOC_HEAD: &str = "blue-moduledoc";
52
53/// The name every doctest's `test` block starts with.
54pub const DOCTEST_PREFIX: &str = "@doc ";
55
56/// The text and definition of a documented top-level form.
57#[must_use]
58pub fn documented(form: &Spanned) -> Option<(&str, &Spanned)> {
59    match form.as_list()? {
60        [head, text, def] if head.as_symbol() == Some(DOC_HEAD) => Some((text.as_string()?, def)),
61        _ => None,
62    }
63}
64
65/// The text of a `@moduledoc` form.
66#[must_use]
67pub fn moduledoc_text(form: &Spanned) -> Option<&str> {
68    match form.as_list()? {
69        [head, text] if head.as_symbol() == Some(MODULEDOC_HEAD) => text.as_string(),
70        _ => None,
71    }
72}
73
74/// The module documentation of a documented parse, with its span.
75#[must_use]
76pub fn module_doc(forms: &[Spanned]) -> Option<(&str, Span)> {
77    forms
78        .iter()
79        .find_map(|f| moduledoc_text(f).map(|t| (t, f.span)))
80}
81
82/// Is `form` a doc node of either kind?
83#[must_use]
84pub fn is_doc_node(form: &Spanned) -> bool {
85    documented(form).is_some() || moduledoc_text(form).is_some()
86}
87
88/// The forms every undocumented consumer sees: a documented definition is
89/// the definition, spanning its documentation; a module doc is dropped.
90#[must_use]
91pub fn strip(forms: Vec<Spanned>) -> Vec<Spanned> {
92    forms
93        .into_iter()
94        .filter(|f| moduledoc_text(f).is_none())
95        .map(|f| match f.form {
96            SpannedForm::List(mut items)
97                if items.len() == 3 && items[0].as_symbol() == Some(DOC_HEAD) =>
98            {
99                let def = items.pop().expect("three items");
100                Spanned::new(f.span, def.form)
101            }
102            form => Spanned::new(f.span, form),
103        })
104        .collect()
105}
106
107/// Put the documentation of `src` back on `forms`, a stripped parse of it
108/// (or a tree built from one whose top-level spans are the parse's): a form
109/// whose span is a documented definition's is wrapped again, its own span
110/// returned to the definition's, and the module doc leads. Forms that are
111/// already documented are returned as they are.
112#[must_use]
113pub fn redocument(src: &str, forms: Vec<Spanned>) -> Vec<Spanned> {
114    if forms.iter().any(is_doc_node) {
115        return forms;
116    }
117    let Ok(documented_forms) = crate::parse_program_documented(src) else {
118        return forms;
119    };
120    let mut out = Vec::with_capacity(forms.len() + 1);
121    for f in &documented_forms {
122        if moduledoc_text(f).is_some() {
123            out.push(f.clone());
124        }
125    }
126    for f in forms {
127        let wrapper = documented_forms
128            .iter()
129            .find(|d| d.span == f.span && documented(d).is_some());
130        match wrapper.and_then(|w| w.as_list()) {
131            Some([head, text, def]) => out.push(Spanned::new(
132                f.span,
133                SpannedForm::List(vec![
134                    head.clone(),
135                    text.clone(),
136                    Spanned::new(def.span, f.form),
137                ]),
138            )),
139            _ => out.push(f),
140        }
141    }
142    out
143}
144
145/// The name a definition binds: `def f`, `defmacro m`, `x = …`.
146#[must_use]
147pub fn defined_name(def: &Spanned) -> Option<&str> {
148    let items = def.as_list()?;
149    if items.first()?.as_symbol()? == "defmacro" {
150        return items.get(1)?.as_symbol();
151    }
152    crate::scope::defined_name_node(items)?.as_symbol()
153}
154
155/// Can `form` carry a `@doc`? A function, macro or top-level value.
156#[must_use]
157pub fn is_definition(form: &Spanned) -> bool {
158    defined_name(form).is_some()
159}
160
161/// One line of an example.
162#[derive(Clone, Debug, PartialEq, Eq)]
163pub enum Step {
164    /// A statement the checks below it depend on.
165    Run(String),
166    /// `expr #=> value`.
167    Check { expr: String, value: String },
168}
169
170/// One example: an indented paragraph of a doc text with at least one
171/// `#=>` line.
172#[derive(Clone, Debug, PartialEq, Eq)]
173pub struct Example {
174    /// Each step with its line in the text, 0-based.
175    pub steps: Vec<(usize, Step)>,
176}
177
178/// Every example in a doc text.
179#[must_use]
180pub fn examples(text: &str) -> Vec<Example> {
181    let mut out = Vec::new();
182    let mut current: Vec<(usize, Step)> = Vec::new();
183    let flush = |current: &mut Vec<(usize, Step)>, out: &mut Vec<Example>| {
184        if current.iter().any(|(_, s)| matches!(s, Step::Check { .. })) {
185            out.push(Example {
186                steps: std::mem::take(current),
187            });
188        } else {
189            current.clear();
190        }
191    };
192    for (i, line) in text.lines().enumerate() {
193        let indented = line.starts_with([' ', '\t']) && !line.trim().is_empty();
194        if !indented {
195            flush(&mut current, &mut out);
196            continue;
197        }
198        let code = line.trim();
199        let step = match code.rsplit_once(" #=> ") {
200            Some((expr, value)) => Step::Check {
201                expr: expr.trim().to_string(),
202                value: value.trim().to_string(),
203            },
204            None => Step::Run(code.to_string()),
205        };
206        current.push((i, step));
207    }
208    flush(&mut current, &mut out);
209    out
210}
211
212/// The intent: the text's first paragraph, its lines joined by spaces.
213#[must_use]
214pub fn intent(text: &str) -> String {
215    text.lines()
216        .take_while(|l| !l.trim().is_empty() && !l.starts_with([' ', '\t']))
217        .map(str::trim)
218        .collect::<Vec<_>>()
219        .join(" ")
220}
221
222/// Is `form` a doctest [`doctests`] made?
223#[must_use]
224pub fn is_doctest(form: &Spanned) -> bool {
225    match form.as_list() {
226        Some([head, name, _]) => {
227            head.as_symbol() == Some("deftest")
228                && name
229                    .as_string()
230                    .is_some_and(|n| n.starts_with(DOCTEST_PREFIX))
231        }
232        _ => false,
233    }
234}
235
236/// Every example of `src`'s documentation as a `test` block, spanned at the
237/// example's lines so a failure names where the example is written.
238///
239/// # Errors
240///
241/// `src` does not parse, or an example's line does not.
242pub fn doctests(src: &str) -> Result<Vec<Spanned>, ParseError> {
243    let forms = crate::parse_program_documented(src)?;
244    let mut out = Vec::new();
245    for form in &forms {
246        let (subject, text, text_span) = if let Some((text, def)) = documented(form) {
247            let Some(name) = defined_name(def) else {
248                continue;
249            };
250            let span = form.as_list().map_or(form.span, |items| items[1].span);
251            (format!("`{name}`"), text, span)
252        } else if let Some(text) = moduledoc_text(form) {
253            ("the module".to_string(), text, form.span)
254        } else {
255            continue;
256        };
257        let mut from = text_span.start;
258        for (k, example) in examples(text).iter().enumerate() {
259            out.push(doctest(
260                src,
261                text,
262                &mut from,
263                text_span.end,
264                &subject,
265                k + 1,
266                example,
267            )?);
268        }
269    }
270    Ok(out)
271}
272
273/// One example as a `test` block whose every node is spanned at the source
274/// it was written as, so a failure, a reference and a rename land on the
275/// example's own characters.
276fn doctest(
277    src: &str,
278    text: &str,
279    from: &mut usize,
280    until: usize,
281    subject: &str,
282    k: usize,
283    example: &Example,
284) -> Result<Spanned, ParseError> {
285    let fail = |at: Span, why: &str| {
286        ParseError::new(format!("{subject}'s example {k} {why}"), at).located(src)
287    };
288    let mut stmts = Vec::new();
289    for (line_no, step) in &example.steps {
290        let written = text.lines().nth(*line_no).unwrap_or("").trim();
291        let at = next_line(src, from, until, written)
292            .ok_or_else(|| fail(Span::new(*from, *from), "is not where its text is"))?;
293        match step {
294            Step::Run(code) => {
295                stmts.push(snippet(src, code, at).map_err(|e| fail(e, "does not parse"))?)
296            }
297            Step::Check { expr, value } => {
298                let marker = at + written.rfind(" #=> ").unwrap_or(0);
299                let after = &written[written.rfind(" #=> ").map_or(0, |i| i + 5)..];
300                let value_at = marker + 5 + (after.len() - after.trim_start().len());
301                let e = snippet(src, expr, at).map_err(|e| fail(e, "does not parse"))?;
302                let v = snippet(src, value, value_at).map_err(|e| fail(e, "does not parse"))?;
303                let op = Span::new(marker + 1, marker + 4);
304                let compared = Span::new(e.span.start, v.span.end);
305                let cmp = Spanned::new(
306                    compared,
307                    SpannedForm::List(vec![symbol(op, "equal?"), e, v]),
308                );
309                let quoted = Spanned::new(compared, SpannedForm::Quote(Box::new(cmp.clone())));
310                stmts.push(Spanned::new(
311                    compared,
312                    SpannedForm::List(vec![symbol(op, crate::LOWERED_ASSERT), quoted, cmp]),
313                ));
314            }
315        }
316    }
317    let first = stmts.first().map_or(*from, |s| s.span.start);
318    let last = stmts.last().map_or(*from, |s| s.span.end);
319    let whole = Span::new(first, last);
320    let body = if stmts.len() == 1 {
321        stmts.remove(0)
322    } else {
323        let mut items = vec![symbol(Span::new(first, first), "begin")];
324        items.extend(stmts);
325        Spanned::new(whole, SpannedForm::List(items))
326    };
327    Ok(Spanned::new(
328        whole,
329        SpannedForm::List(vec![
330            symbol(Span::new(first, first), "deftest"),
331            text_node(
332                Span::new(first, first),
333                format!("{DOCTEST_PREFIX}{subject} example {k}"),
334            ),
335            body,
336        ]),
337    ))
338}
339
340/// Where the next source line reading `written` starts its code, at or
341/// after `from` and before `until`; `from` moves past it.
342fn next_line(src: &str, from: &mut usize, until: usize, written: &str) -> Option<usize> {
343    let region = src.get(*from..until.min(src.len()))?;
344    let mut offset = *from;
345    for line in region.split_inclusive('\n') {
346        if line.trim() == written {
347            let code = offset + (line.len() - line.trim_start().len());
348            *from = offset + line.len();
349            return Some(code);
350        }
351        offset += line.len();
352    }
353    None
354}
355
356/// One expression or statement of an example, parsed as part of its text
357/// and spanned where it is written.
358fn snippet(src: &str, code: &str, at: usize) -> Result<Spanned, Span> {
359    let here = Span::new(at, at + code.len());
360    let mut forms = crate::parse::parse_snippet(code).map_err(|_| here)?;
361    if forms.len() != 1 || src.get(here.start..here.end) != Some(code) {
362        return Err(here);
363    }
364    Ok(shift(forms.remove(0), at))
365}
366
367fn symbol(span: Span, name: &str) -> Spanned {
368    Spanned::new(span, SpannedForm::Atom(Atom::Symbol(name.into())))
369}
370
371fn shift(node: Spanned, by: usize) -> Spanned {
372    let span = if node.span.is_synthetic() {
373        node.span
374    } else {
375        Span::new(node.span.start + by, node.span.end + by)
376    };
377    let form = match node.form {
378        SpannedForm::List(items) => {
379            SpannedForm::List(items.into_iter().map(|i| shift(i, by)).collect())
380        }
381        SpannedForm::Quote(x) => SpannedForm::Quote(Box::new(shift(*x, by))),
382        SpannedForm::Quasiquote(x) => SpannedForm::Quasiquote(Box::new(shift(*x, by))),
383        SpannedForm::Unquote(x) => SpannedForm::Unquote(Box::new(shift(*x, by))),
384        SpannedForm::UnquoteSplice(x) => SpannedForm::UnquoteSplice(Box::new(shift(*x, by))),
385        other => other,
386    };
387    Spanned::new(span, form)
388}
389
390/// A doc text node, as the parser builds it.
391pub(crate) fn text_node(span: Span, text: String) -> Spanned {
392    Spanned::new(span, SpannedForm::Atom(Atom::Str(text.into())))
393}
394
395#[cfg(test)]
396mod tests {
397    use super::*;
398
399    const SRC: &str = "@moduledoc \"\"\"\n  m does a thing.\n\n      f(1) #=> 2\n  \"\"\"\n\n@doc \"\"\"\nf adds one.\n\n    x = 1\n    f(x) #=> 2\n    f(2) #=> 3\n\"\"\"\ndef f(n)\n  n + 1\nend\n";
400
401    #[test]
402    fn the_documented_tree_holds_both_kinds_and_the_stripped_one_neither() {
403        let doc = crate::parse_program_documented(SRC).unwrap();
404        assert_eq!(doc.len(), 2);
405        assert_eq!(
406            module_doc(&doc).map(|(t, _)| t),
407            Some("m does a thing.\n\n    f(1) #=> 2")
408        );
409        let (text, def) = documented(&doc[1]).unwrap();
410        assert!(text.starts_with("f adds one."));
411        assert_eq!(defined_name(def), Some("f"));
412        let plain = crate::parse_program_tree(SRC).unwrap();
413        assert_eq!(plain.len(), 1);
414        assert_eq!(plain[0].to_sexp().to_string(), "(define (f n) (+ n 1))");
415        assert_eq!(plain[0].span, doc[1].span);
416    }
417
418    #[test]
419    fn redocument_restores_what_strip_took() {
420        let doc = crate::parse_program_documented(SRC).unwrap();
421        let plain = crate::parse_program_tree(SRC).unwrap();
422        assert_eq!(redocument(SRC, plain), doc);
423    }
424
425    #[test]
426    fn examples_and_intent_are_read_from_the_text() {
427        let text = "f adds one,\nto a number.\n\n    x = 1\n    f(x) #=> 2\n\nprose\n\n    g(1)\n";
428        assert_eq!(intent(text), "f adds one, to a number.");
429        let ex = examples(text);
430        assert_eq!(ex.len(), 1, "a paragraph with no `#=>` is not an example");
431        assert_eq!(
432            ex[0].steps,
433            vec![
434                (3, Step::Run("x = 1".into())),
435                (
436                    4,
437                    Step::Check {
438                        expr: "f(x)".into(),
439                        value: "2".into()
440                    }
441                )
442            ]
443        );
444    }
445
446    #[test]
447    fn each_example_is_one_test_spanned_at_its_lines() {
448        let tests = doctests(SRC).unwrap();
449        assert_eq!(tests.len(), 2);
450        assert!(tests.iter().all(is_doctest));
451        assert_eq!(
452            tests[1].to_sexp().to_string(),
453            "(deftest \"@doc `f` example 1\" (begin (define x 1) (blue-assert '(equal? (f x) 2) (equal? (f x) 2)) (blue-assert '(equal? (f 2) 3) (equal? (f 2) 3))))"
454        );
455        let at = tests[1].span;
456        assert_eq!(
457            &SRC[at.start..at.end],
458            "x = 1\n    f(x) #=> 2\n    f(2) #=> 3"
459        );
460        fn refs(n: &Spanned, out: &mut Vec<Span>) {
461            if n.as_symbol() == Some("f") {
462                out.push(n.span);
463            }
464            if let Some(items) = n.as_list() {
465                for i in items {
466                    refs(i, out);
467                }
468            }
469        }
470        let mut found = Vec::new();
471        refs(&tests[1], &mut found);
472        assert!(!found.is_empty());
473        for s in found {
474            assert_eq!(&SRC[s.start..s.end], "f");
475        }
476    }
477
478    #[test]
479    fn a_doc_must_document_a_definition_and_a_moduledoc_must_lead() {
480        let e = crate::parse_program_documented("@doc \"\"\"\nx\n\"\"\"\nf(1)\n").unwrap_err();
481        assert!(
482            e.message.contains("documents the definition below it"),
483            "{e}"
484        );
485        let e =
486            crate::parse_program_documented("x = 1\n@moduledoc \"\"\"\nm\n\"\"\"\n").unwrap_err();
487        assert!(e.message.contains("first form"), "{e}");
488        let e =
489            crate::parse_program_documented("def f()\n  @doc \"\"\"\n  x\n  \"\"\"\n  1\nend\n")
490                .unwrap_err();
491        assert!(e.message.contains("top level"), "{e}");
492    }
493}
blue_lang_syntax/doc.rs

blue_lang_syntax/
doc.rs
