Skip to main content

fatou_parser/ast/
documentation.rs

1//! Typed documentation attachments and statically recoverable textual payloads.
2//!
3//! Julia's `@doc` accepts arbitrary metadata. This module therefore recognizes
4//! attachment shape independently from content and decodes only standard
5//! ordinary or `raw` string literals whose value is statically knowable.
6
7use rowan::ast::AstNode as _;
8use rowan::{TextRange, TextSize};
9
10use super::{Doc, MacroCall, StringLiteral};
11use crate::parser::StringDecodeError;
12use crate::syntax::{SyntaxKind, SyntaxNode};
13
14/// Whether documentation used Julia's adjacent-string sugar or an explicit
15/// two-argument `@doc` macro call.
16#[derive(Debug, Clone, Copy, PartialEq, Eq)]
17pub enum DocAttachmentKind {
18    Juxtaposed,
19    Macro,
20}
21
22/// A documentable expression or statement.
23///
24/// Julia permits unrelated syntax kinds here, so this wrapper exposes the
25/// lossless node without claiming that every target is an [`Expr`](super::Expr).
26#[derive(Debug, Clone, PartialEq, Eq, Hash)]
27pub struct DocTarget(SyntaxNode);
28
29impl DocTarget {
30    pub(crate) fn new(syntax: SyntaxNode) -> Self {
31        Self(syntax)
32    }
33
34    /// Return the lossless syntax node for the documented target.
35    pub fn syntax(&self) -> &SyntaxNode {
36        &self.0
37    }
38}
39
40/// A documentation payload and the syntax it documents.
41///
42/// This is deliberately not an [`AstNode`](rowan::ast::AstNode): it unifies a
43/// `DOC` node with qualified or unqualified spellings of a two-argument `@doc`
44/// `MACRO_CALL`.
45#[derive(Debug, Clone, PartialEq, Eq)]
46pub struct DocAttachment {
47    syntax: SyntaxNode,
48    payload: SyntaxNode,
49    target: SyntaxNode,
50    kind: DocAttachmentKind,
51}
52
53impl DocAttachment {
54    /// Recognize an adjacent docstring or exactly two arguments to a macro whose
55    /// final identifier is `doc` (`@doc`, `Base.@doc`, or `@Base.doc`).
56    pub fn cast(syntax: SyntaxNode) -> Option<Self> {
57        if let Some(doc) = Doc::cast(syntax.clone()) {
58            return Some(Self {
59                payload: doc.literal()?.syntax().clone(),
60                target: doc.target()?.syntax().clone(),
61                syntax,
62                kind: DocAttachmentKind::Juxtaposed,
63            });
64        }
65
66        let call = MacroCall::cast(syntax.clone())?;
67        let name = call.name()?;
68        if name.macro_token()?.text() != "doc" {
69            return None;
70        }
71        let args = macro_args(call.syntax());
72        let [payload, target] = args.as_slice() else {
73            return None;
74        };
75        Some(Self {
76            syntax,
77            payload: payload.clone(),
78            target: target.clone(),
79            kind: DocAttachmentKind::Macro,
80        })
81    }
82
83    /// Return the complete attachment node.
84    pub fn syntax(&self) -> &SyntaxNode {
85        &self.syntax
86    }
87
88    /// Return the metadata expression passed as documentation.
89    pub fn payload(&self) -> &SyntaxNode {
90        &self.payload
91    }
92
93    /// Return the expression or statement that receives the documentation.
94    pub fn target(&self) -> &SyntaxNode {
95        &self.target
96    }
97
98    /// Return the Julia syntax form used for the attachment.
99    pub fn kind(&self) -> DocAttachmentKind {
100        self.kind
101    }
102
103    /// Recover textual documentation when Julia's value is statically known.
104    pub fn text(&self) -> DocText {
105        let Some(literal) = StringLiteral::cast(self.payload.clone()) else {
106            return DocText::Opaque(OpaqueDocReason::NonString);
107        };
108        decode_literal(&literal)
109    }
110}
111
112/// Why a documentation payload cannot be interpreted statically.
113#[derive(Debug, Clone, Copy, PartialEq, Eq)]
114pub enum OpaqueDocReason {
115    /// The payload is not a string literal.
116    NonString,
117    /// The string literal contains interpolation.
118    Interpolation,
119    /// The string macro is not Julia's standard `raw` string.
120    UnsupportedPrefix,
121    /// The literal has a suffix, so a string macro controls its meaning.
122    UnsupportedSuffix,
123}
124
125/// The statically knowable status of a documentation payload.
126#[derive(Debug, Clone, PartialEq, Eq)]
127pub enum DocText {
128    /// The exact documentation value is statically recoverable.
129    Static(StaticDocText),
130    /// Julia syntax permits the payload, but its value requires evaluation or
131    /// string-macro semantics.
132    Opaque(OpaqueDocReason),
133    /// A standard string literal contains an invalid escape or byte sequence.
134    Invalid(StringDecodeError),
135}
136
137/// Decoded documentation text and its mapping back to Julia source bytes.
138#[derive(Debug, Clone, PartialEq, Eq)]
139pub struct StaticDocText {
140    text: String,
141    source_map: DocSourceMap,
142}
143
144impl StaticDocText {
145    /// Return the decoded documentation value.
146    pub fn as_str(&self) -> &str {
147        &self.text
148    }
149
150    /// Return the mapping from decoded bytes to Julia source bytes.
151    pub fn source_map(&self) -> &DocSourceMap {
152        &self.source_map
153    }
154}
155
156/// An exact byte-span map from decoded documentation to the string literal.
157///
158/// The map is stored as runs rather than one span per decoded byte: a
159/// docstring is overwhelmingly literal text whose spans advance one source
160/// byte at a time, so the common case is a single run. Size matters because a
161/// whole file's maps live in the semantic model, which salsa caches and
162/// compares for backdating.
163#[derive(Debug, Clone, PartialEq, Eq)]
164pub struct DocSourceMap {
165    runs: Vec<SourceRun>,
166    len: u32,
167    empty_at: TextSize,
168}
169
170impl DocSourceMap {
171    /// Map a decoded UTF-8 byte range to its enclosing absolute source range.
172    /// Returns `None` for an out-of-bounds range.
173    pub fn source_range(&self, range: TextRange) -> Option<TextRange> {
174        let start = u32::from(range.start());
175        let end = u32::from(range.end());
176        if start > end || end > self.len {
177            return None;
178        }
179        if start == end {
180            let at = self
181                .span(start)
182                .map(|span| span.start)
183                .or_else(|| self.runs.last().map(|run| run.span(run.len - 1).end))
184                .unwrap_or(self.empty_at);
185            return Some(TextRange::new(at, at));
186        }
187        Some(TextRange::new(
188            self.span(start)?.start,
189            self.span(end - 1)?.end,
190        ))
191    }
192
193    /// Map an absolute Julia source offset into the decoded documentation.
194    ///
195    /// A cursor anywhere inside an escape maps to the first decoded byte that
196    /// escape produced. Bytes removed by triple-string dedenting or a line
197    /// continuation have no decoded position and return `None`.
198    pub fn decoded_offset(&self, offset: TextSize) -> Option<TextSize> {
199        let Some(last) = self.runs.last() else {
200            return (offset == self.empty_at).then_some(TextSize::new(0));
201        };
202        if offset == last.span(last.len - 1).end {
203            return Some(TextSize::new(self.len));
204        }
205        // Runs cover the source in increasing order, so the one that could
206        // contain `offset` is the last starting at or before it. Dedented and
207        // continued bytes fall in the gaps between runs.
208        let index = self
209            .runs
210            .partition_point(|run| run.source_start <= offset)
211            .checked_sub(1)?;
212        let run = &self.runs[index];
213        let step = run.step(offset)?;
214        // Several decoded bytes can share one span (a multi-byte `\u` escape);
215        // the cursor belongs to the first of them.
216        let first = if run.stride == 0 { 0 } else { step };
217        Some(TextSize::new(run.decoded_start + first))
218    }
219
220    /// The span of one decoded byte, or `None` past the end of the map.
221    fn span(&self, decoded: u32) -> Option<SourceSpan> {
222        let index = self
223            .runs
224            .partition_point(|run| run.decoded_start <= decoded)
225            .checked_sub(1)?;
226        let run = &self.runs[index];
227        (decoded - run.decoded_start < run.len).then(|| run.span(decoded - run.decoded_start))
228    }
229}
230
231/// A maximal span sequence advancing by a fixed source stride.
232///
233/// `stride` is zero when every decoded byte in the run shares one source span,
234/// one for ordinary literal text, and larger for a run of same-width escapes.
235#[derive(Debug, Clone, Copy, PartialEq, Eq)]
236struct SourceRun {
237    decoded_start: u32,
238    len: u32,
239    source_start: TextSize,
240    stride: u32,
241    width: u32,
242}
243
244impl SourceRun {
245    fn span(&self, step: u32) -> SourceSpan {
246        let start = u32::from(self.source_start) + step * self.stride;
247        SourceSpan {
248            start: TextSize::new(start),
249            end: TextSize::new(start + self.width),
250        }
251    }
252
253    /// The step within this run whose span contains `offset`.
254    fn step(&self, offset: TextSize) -> Option<u32> {
255        let relative = u32::from(offset).checked_sub(u32::from(self.source_start))?;
256        let step = relative
257            .checked_div(self.stride)
258            .map_or(0, |step| step.min(self.len - 1));
259        let span = self.span(step);
260        (span.start <= offset && offset < span.end).then_some(step)
261    }
262
263    /// Extend the run with `span` when it continues the same stride and width.
264    fn extend(&mut self, span: SourceSpan) -> bool {
265        if u32::from(span.end) - u32::from(span.start) != self.width {
266            return false;
267        }
268        if self.len == 1 {
269            // A one-byte run has no stride yet, so any successor defines it.
270            let Some(stride) = u32::from(span.start).checked_sub(u32::from(self.source_start))
271            else {
272                return false;
273            };
274            self.stride = stride;
275            self.len += 1;
276            return true;
277        }
278        if span.start != self.span(self.len).start {
279            return false;
280        }
281        self.len += 1;
282        true
283    }
284}
285
286#[derive(Debug, Clone, Copy, PartialEq, Eq)]
287struct SourceSpan {
288    start: TextSize,
289    end: TextSize,
290}
291
292/// Compress a per-decoded-byte span vector into runs.
293fn compress(bytes: &[SourceSpan]) -> Vec<SourceRun> {
294    let mut runs: Vec<SourceRun> = Vec::new();
295    for (index, span) in bytes.iter().enumerate() {
296        if let Some(run) = runs.last_mut()
297            && run.extend(*span)
298        {
299            continue;
300        }
301        runs.push(SourceRun {
302            decoded_start: index as u32,
303            len: 1,
304            source_start: span.start,
305            stride: 1,
306            width: u32::from(span.end) - u32::from(span.start),
307        });
308    }
309    runs
310}
311
312#[derive(Clone)]
313struct MappedText {
314    text: String,
315    bytes: Vec<SourceSpan>,
316    empty_at: TextSize,
317}
318
319impl MappedText {
320    fn from_literal(literal: &StringLiteral) -> Self {
321        let empty_at = literal
322            .syntax()
323            .children_with_tokens()
324            .filter_map(|element| element.into_token())
325            .find(|token| token.kind() == SyntaxKind::STRING_DELIM_OPEN)
326            .map(|token| token.text_range().end())
327            .unwrap_or_else(|| literal.syntax().text_range().start());
328        let mut out = Self {
329            text: String::new(),
330            bytes: Vec::new(),
331            empty_at,
332        };
333        for token in literal.content_tokens() {
334            let base: u32 = token.text_range().start().into();
335            out.text.push_str(token.text());
336            for i in 0..token.text().len() as u32 {
337                out.bytes.push(SourceSpan {
338                    start: TextSize::from(base + i),
339                    end: TextSize::from(base + i + 1),
340                });
341            }
342        }
343        out
344    }
345
346    fn push_slice(&mut self, source: &Self, range: std::ops::Range<usize>) {
347        self.text.push_str(&source.text[range.clone()]);
348        self.bytes.extend_from_slice(&source.bytes[range]);
349    }
350
351    fn into_static(self) -> StaticDocText {
352        StaticDocText {
353            text: self.text,
354            source_map: DocSourceMap {
355                runs: compress(&self.bytes),
356                len: self.bytes.len() as u32,
357                empty_at: self.empty_at,
358            },
359        }
360    }
361}
362
363fn macro_args(node: &SyntaxNode) -> Vec<SyntaxNode> {
364    let mut out = Vec::new();
365    for child in node.children() {
366        match child.kind() {
367            SyntaxKind::MACRO_NAME => {}
368            SyntaxKind::ARG_LIST => {
369                for arg in child.children() {
370                    if arg.kind() == SyntaxKind::ARG {
371                        out.extend(arg.children());
372                    } else {
373                        out.push(arg);
374                    }
375                }
376            }
377            _ => out.push(child),
378        }
379    }
380    out
381}
382
383fn decode_literal(literal: &StringLiteral) -> DocText {
384    if literal.interpolations().next().is_some() {
385        return DocText::Opaque(OpaqueDocReason::Interpolation);
386    }
387    if literal.suffix().is_some() {
388        return DocText::Opaque(OpaqueDocReason::UnsupportedSuffix);
389    }
390    let raw = match literal.prefix().as_ref().map(|token| token.text()) {
391        None => false,
392        Some("raw") => true,
393        Some(_) => return DocText::Opaque(OpaqueDocReason::UnsupportedPrefix),
394    };
395    let triple = literal
396        .syntax()
397        .children_with_tokens()
398        .filter_map(|element| element.into_token())
399        .find(|token| token.kind() == SyntaxKind::STRING_DELIM_OPEN)
400        .is_some_and(|token| token.text().len() >= 3);
401
402    let mut mapped = MappedText::from_literal(literal);
403    if raw {
404        mapped = decode_raw(mapped);
405    }
406    if triple {
407        mapped = dedent_triple(normalize_newlines(mapped));
408    }
409    if !raw {
410        mapped = match decode_escapes(mapped) {
411            Ok(mapped) => mapped,
412            Err(err) => return DocText::Invalid(err),
413        };
414    }
415    DocText::Static(mapped.into_static())
416}
417
418fn normalize_newlines(input: MappedText) -> MappedText {
419    let bytes = input.text.as_bytes();
420    let mut out = MappedText {
421        text: String::new(),
422        bytes: Vec::new(),
423        empty_at: input.empty_at,
424    };
425    let mut i = 0;
426    while i < bytes.len() {
427        if bytes[i] == b'\r' {
428            let end = if bytes.get(i + 1) == Some(&b'\n') {
429                i + 2
430            } else {
431                i + 1
432            };
433            out.text.push('\n');
434            out.bytes.push(SourceSpan {
435                start: input.bytes[i].start,
436                end: input.bytes[end - 1].end,
437            });
438            i = end;
439        } else {
440            let len = input.text[i..].chars().next().unwrap().len_utf8();
441            out.push_slice(&input, i..i + len);
442            i += len;
443        }
444    }
445    out
446}
447
448fn dedent_triple(input: MappedText) -> MappedText {
449    let mut lines = Vec::new();
450    let mut start = 0;
451    for (i, byte) in input.text.bytes().enumerate() {
452        if byte == b'\n' {
453            lines.push((start, i, Some(i)));
454            start = i + 1;
455        }
456    }
457    lines.push((start, input.text.len(), None));
458
459    let last = lines.len() - 1;
460    let candidates: Vec<&str> = lines
461        .iter()
462        .enumerate()
463        .filter(|(i, (start, end, _))| {
464            *i != 0
465                && (*i == last
466                    || !input.text[*start..*end]
467                        .bytes()
468                        .all(|byte| matches!(byte, b' ' | b'\t')))
469        })
470        .map(|(_, (start, end, _))| {
471            let line = &input.text[*start..*end];
472            &line[..line
473                .bytes()
474                .take_while(|byte| matches!(byte, b' ' | b'\t'))
475                .count()]
476        })
477        .collect();
478    let dedent = common_whitespace_prefix_len(&candidates);
479
480    let mut out = MappedText {
481        text: String::new(),
482        bytes: Vec::new(),
483        empty_at: input.empty_at,
484    };
485    for (i, (start, end, newline)) in lines.into_iter().enumerate() {
486        if i == 0 && start == end {
487            continue;
488        }
489        let strip = if i == 0 { 0 } else { dedent.min(end - start) };
490        out.push_slice(&input, start + strip..end);
491        if let Some(newline) = newline {
492            out.push_slice(&input, newline..newline + 1);
493        }
494    }
495    out
496}
497
498/// Return the common byte prefix of leading space/tab strings.
499///
500/// This is shared with the JuliaSyntax projector so the public docstring value
501/// and the differential oracle cannot drift on triple-string indentation.
502pub(crate) fn common_whitespace_prefix_len(strings: &[&str]) -> usize {
503    let Some(first) = strings.first() else {
504        return 0;
505    };
506    strings.iter().skip(1).fold(first.len(), |len, string| {
507        first
508            .bytes()
509            .zip(string.bytes())
510            .take(len)
511            .take_while(|(a, b)| a == b)
512            .count()
513    })
514}
515
516fn decode_raw(input: MappedText) -> MappedText {
517    let bytes = input.text.as_bytes();
518    let mut out = MappedText {
519        text: String::new(),
520        bytes: Vec::new(),
521        empty_at: input.empty_at,
522    };
523    let mut i = 0;
524    while i < bytes.len() {
525        if bytes[i] != b'\\' {
526            let len = input.text[i..].chars().next().unwrap().len_utf8();
527            out.push_slice(&input, i..i + len);
528            i += len;
529            continue;
530        }
531        let mut run = 0;
532        while i + run < bytes.len() && bytes[i + run] == b'\\' {
533            run += 1;
534        }
535        let at_end = i + run == bytes.len();
536        let before_quote = bytes.get(i + run) == Some(&b'"');
537        if !at_end && !before_quote {
538            out.push_slice(&input, i..i + run);
539            i += run;
540            continue;
541        }
542        for pair in 0..run / 2 {
543            let start = i + pair * 2;
544            out.text.push('\\');
545            out.bytes.push(SourceSpan {
546                start: input.bytes[start].start,
547                end: input.bytes[start + 1].end,
548            });
549        }
550        i += run;
551        if run % 2 == 1 {
552            out.text.push('"');
553            let end = if before_quote { i + 1 } else { i };
554            out.bytes.push(SourceSpan {
555                start: input.bytes[i - 1].start,
556                end: input.bytes[end.saturating_sub(1)].end,
557            });
558            i = end;
559        }
560    }
561    out
562}
563
564fn decode_escapes(input: MappedText) -> Result<MappedText, StringDecodeError> {
565    let source = input.text.as_bytes();
566    let mut decoded = Vec::new();
567    let mut spans = Vec::new();
568    let mut i = 0;
569    while i < source.len() {
570        if source[i] != b'\\' {
571            let len = input.text[i..].chars().next().unwrap().len_utf8();
572            decoded.extend_from_slice(&source[i..i + len]);
573            spans.extend_from_slice(&input.bytes[i..i + len]);
574            i += len;
575            continue;
576        }
577        let start = i;
578        i += 1;
579        let Some(&escaped) = source.get(i) else {
580            return Err(StringDecodeError::BadEscape);
581        };
582        if matches!(escaped, b'\n' | b'\r') {
583            if escaped == b'\r' && source.get(i + 1) == Some(&b'\n') {
584                i += 1;
585            }
586            i += 1;
587            while matches!(source.get(i), Some(b' ' | b'\t')) {
588                i += 1;
589            }
590            continue;
591        }
592        let value = match escaped {
593            b'n' => vec![b'\n'],
594            b't' => vec![b'\t'],
595            b'r' => vec![b'\r'],
596            b'a' => vec![0x07],
597            b'b' => vec![0x08],
598            b'f' => vec![0x0c],
599            b'v' => vec![0x0b],
600            b'e' => vec![0x1b],
601            b'\\' => vec![b'\\'],
602            b'\'' => vec![b'\''],
603            b'"' => vec![b'"'],
604            b'$' => vec![b'$'],
605            b'x' => decode_numeric_escape(source, &mut i, 2, false)?,
606            b'u' => decode_numeric_escape(source, &mut i, 4, true)?,
607            b'U' => decode_numeric_escape(source, &mut i, 8, true)?,
608            b'0'..=b'7' => decode_octal_escape(source, &mut i)?,
609            _ => return Err(StringDecodeError::BadEscape),
610        };
611        i += 1;
612        let span = SourceSpan {
613            start: input.bytes[start].start,
614            end: input.bytes[i - 1].end,
615        };
616        decoded.extend_from_slice(&value);
617        spans.extend(std::iter::repeat_n(span, value.len()));
618    }
619    let text = String::from_utf8(decoded).map_err(|_| StringDecodeError::BadUtf8)?;
620    Ok(MappedText {
621        text,
622        bytes: spans,
623        empty_at: input.empty_at,
624    })
625}
626
627fn decode_numeric_escape(
628    source: &[u8],
629    i: &mut usize,
630    max: usize,
631    unicode: bool,
632) -> Result<Vec<u8>, StringDecodeError> {
633    let mut value = 0u32;
634    let mut count = 0;
635    while count < max {
636        let Some(digit) = source
637            .get(*i + 1)
638            .and_then(|byte| char::from(*byte).to_digit(16))
639        else {
640            break;
641        };
642        *i += 1;
643        value = value * 16 + digit;
644        count += 1;
645    }
646    if count == 0 {
647        return Err(StringDecodeError::BadEscape);
648    }
649    if !unicode {
650        return Ok(vec![value as u8]);
651    }
652    let character = char::from_u32(value).ok_or(StringDecodeError::BadEscape)?;
653    let mut buffer = [0; 4];
654    Ok(character.encode_utf8(&mut buffer).as_bytes().to_vec())
655}
656
657fn decode_octal_escape(source: &[u8], i: &mut usize) -> Result<Vec<u8>, StringDecodeError> {
658    let mut value = u32::from(source[*i] - b'0');
659    for _ in 0..2 {
660        let Some(digit) = source
661            .get(*i + 1)
662            .and_then(|byte| char::from(*byte).to_digit(8))
663        else {
664            break;
665        };
666        *i += 1;
667        value = value * 8 + digit;
668    }
669    let byte = u8::try_from(value).map_err(|_| StringDecodeError::BadEscape)?;
670    Ok(vec![byte])
671}