weavatrix_parse/docs/
mod.rs

1//! Structural extraction for Markdown, MDX, `reStructuredText` and `AsciiDoc`.
2//!
3//! Prose has no token structure worth the name: a `"` is a quotation mark, not
4//! a literal, and `//` is part of a URL. So this reads lines directly rather
5//! than through the tokenizer, which is the honest model for the format even
6//! though it is the opposite of what every other extractor here does.
7//!
8//! Two facts are worth having. A heading is a named anchor other documents
9//! link to, and nesting one heading under another is the document's own table
10//! of contents. A link to a path in the repository is a dependency exactly as
11//! an import is - which is what turns a documentation tree into part of the
12//! graph rather than a pile of files beside it.
13
14use crate::facts::{Declaration, DeclarationKind, Facts, Import, Span};
15use crate::syntax::Language;
16
17/// Extracts structural facts from one document.
18#[must_use]
19pub fn extract(source: &str, language: Language) -> Facts {
20    let mut state = Extractor {
21        facts: Facts::default(),
22        headings: Vec::new(),
23        offset: 0,
24    };
25    state.run(source, language);
26    state.facts
27}
28
29/// A heading whose section later headings may nest inside.
30struct Heading {
31    name: String,
32    level: usize,
33}
34
35struct Extractor {
36    facts: Facts,
37    headings: Vec<Heading>,
38    offset: usize,
39}
40
41impl Extractor {
42    fn run(&mut self, source: &str, language: Language) {
43        let lines = source.lines().collect::<Vec<_>>();
44        let mut fenced = false;
45        let mut script = String::new();
46        for (number, line) in lines.iter().enumerate() {
47            let start = self.offset;
48            self.offset += line.len() + 1;
49            let trimmed = line.trim();
50            // A fence hides everything until it closes: code inside a block is
51            // an example, and reading its links would invent dependencies.
52            if trimmed.starts_with("```") || trimmed.starts_with("~~~") {
53                fenced = !fenced;
54                continue;
55            }
56            if fenced {
57                continue;
58            }
59            let span = Span {
60                start,
61                end: start + line.len(),
62                line: u32::try_from(number + 1).unwrap_or(u32::MAX),
63                column: 1,
64                end_line: u32::try_from(number + 1).unwrap_or(u32::MAX),
65                end_column: u32::try_from(line.len() + 1).unwrap_or(u32::MAX),
66            };
67            // MDX holds real JavaScript imports, which the script extractor
68            // already reads correctly; gathering them keeps that one rule.
69            if language == Language::Mdx
70                && (trimmed.starts_with("import ") || trimmed.starts_with("export "))
71            {
72                script.push_str(line);
73                script.push('\n');
74                continue;
75            }
76            self.heading(trimmed, lines.get(number + 1).copied(), language, span);
77            self.targets(line, language, span);
78        }
79        if !script.is_empty() {
80            let inner = crate::script::extract(&script, Language::TypeScript);
81            self.facts.imports.extend(inner.imports);
82        }
83    }
84
85    /// Records a heading and nests it under the last shallower one.
86    fn heading(&mut self, line: &str, next: Option<&str>, language: Language, span: Span) {
87        let Some((level, name)) = read_heading(line, next, language) else {
88            return;
89        };
90        while self
91            .headings
92            .last()
93            .is_some_and(|heading| heading.level >= level)
94        {
95            self.headings.pop();
96        }
97        self.facts.declarations.push(Declaration {
98            name: name.clone(),
99            kind: DeclarationKind::Heading,
100            span,
101            extent: span,
102            owner: self.headings.last().map(|heading| heading.name.clone()),
103            // Every heading in a document is reachable by anchor.
104            exported: true,
105        });
106        self.headings.push(Heading { name, level });
107    }
108
109    /// Records every path this line points at.
110    fn targets(&mut self, line: &str, language: Language, span: Span) {
111        for target in link_targets(line, language) {
112            if !is_repository_path(&target) {
113                continue;
114            }
115            self.facts.imports.push(Import {
116                specifier: target,
117                span,
118                type_only: false,
119                reexport: false,
120                names: Vec::new(),
121                bindings: Vec::new(),
122            });
123        }
124    }
125}
126
127mod formats;
128
129use formats::{is_repository_path, link_targets, read_heading};
130
131#[cfg(test)]
132mod tests;
weavatrix_parse/docs/mod.rs

weavatrix_parse/docs/
mod.rs
