diffler_core/syntax/
symbols.rs

1//! Every identifier in a file, by line and byte range.
2
3use std::ops::Range;
4
5use crate::syntax::registry::LanguageRegistry;
6use crate::syntax::scope::tag_pass;
7use crate::syntax::{MAX_PARSE_BYTES, ScopeIndex, parse};
8
9#[derive(Debug, Clone, PartialEq, Eq)]
10pub struct Ident {
11    pub line: usize,
12    pub range: Range<usize>,
13    pub name: String,
14    /// A function, method or type: tagged by the grammar's tags query, or in
15    /// a type position.
16    pub item: bool,
17}
18
19impl LanguageRegistry {
20    /// Every identifier in `content` and its definition spans, from one
21    /// parse. Without a parse we fall back to plain words, strings and
22    /// comments included.
23    pub fn symbols(&self, path: &str, content: &str) -> (Vec<Ident>, ScopeIndex) {
24        let entry = (content.len() <= MAX_PARSE_BYTES)
25            .then(|| self.for_file(path, content))
26            .flatten();
27        let Some((entry, tree)) = entry.and_then(|entry| Some((entry, parse(entry, content)?)))
28        else {
29            return (words(content), ScopeIndex::default());
30        };
31        let (scope, tagged) = entry
32            .tags()
33            .map(|query| tag_pass(query, &tree, content))
34            .unwrap_or_default();
35        let mut idents = Vec::new();
36        let mut cursor = tree.walk();
37        loop {
38            let node = cursor.node();
39            if node.child_count() == 0 && !node.is_missing() && is_identifier(node.kind()) {
40                let start = node.start_position();
41                let len = node.end_byte() - node.start_byte();
42                if let Some(name) = content
43                    .get(node.start_byte()..node.end_byte())
44                    .filter(|name| !name.is_empty())
45                {
46                    idents.push(Ident {
47                        line: start.row,
48                        range: start.column..start.column + len,
49                        name: name.to_owned(),
50                        item: node.kind().contains("type") || tagged.contains(&node.start_byte()),
51                    });
52                }
53            }
54            if cursor.goto_first_child() || cursor.goto_next_sibling() {
55                continue;
56            }
57            loop {
58                if !cursor.goto_parent() {
59                    return (idents, scope);
60                }
61                if cursor.goto_next_sibling() {
62                    break;
63                }
64            }
65        }
66    }
67}
68
69/// PHP spells an identifier `name` and Ruby `constant`.
70fn is_identifier(kind: &str) -> bool {
71    kind.ends_with("identifier") || kind == "name" || kind == "constant"
72}
73
74/// Every run of `[A-Za-z0-9_]` that does not start with a digit.
75fn words(content: &str) -> Vec<Ident> {
76    let mut out = Vec::new();
77    for (line, text) in content.lines().enumerate() {
78        let mut start = None;
79        for (at, ch) in text
80            .char_indices()
81            .chain(std::iter::once((text.len(), ' ')))
82        {
83            let word_char = ch.is_ascii_alphanumeric() || ch == '_';
84            match (start, word_char) {
85                (None, true) => start = Some(at),
86                (Some(from), false) => {
87                    start = None;
88                    let Some(token) = text.get(from..at) else {
89                        continue;
90                    };
91                    if !token.starts_with(|first: char| first.is_ascii_digit()) {
92                        out.push(Ident {
93                            line,
94                            range: from..at,
95                            name: token.to_owned(),
96                            item: false,
97                        });
98                    }
99                }
100                _ => {}
101            }
102        }
103    }
104    out
105}
106
107#[cfg(test)]
108mod tests {
109    use super::*;
110
111    fn names_on(idents: &[Ident], line: usize) -> Vec<&str> {
112        idents
113            .iter()
114            .filter(|ident| ident.line == line)
115            .map(|ident| ident.name.as_str())
116            .collect()
117    }
118
119    #[test]
120    fn a_parsed_file_yields_its_names_and_skips_strings_and_comments() {
121        let reg = LanguageRegistry::build();
122        let src = "fn total(price: u32, qty: u32) -> u32 {\n    // price times qty\n    let label = \"qty\";\n    price * qty\n}\n";
123        let idents = reg.symbols("a.rs", src).0;
124        assert_eq!(
125            names_on(&idents, 0),
126            ["total", "price", "qty"],
127            "the grammar parses a primitive type as its own node"
128        );
129        assert!(names_on(&idents, 1).is_empty(), "a comment names nothing");
130        assert_eq!(names_on(&idents, 2), ["label"], "a string names nothing");
131        assert_eq!(names_on(&idents, 3), ["price", "qty"]);
132        let price = idents.iter().find(|i| i.line == 3).expect("price");
133        assert_eq!(
134            &src.lines().nth(3).expect("line")[price.range.clone()],
135            "price"
136        );
137    }
138
139    #[test]
140    fn a_call_or_a_definition_names_an_item_and_a_value_does_not() {
141        let reg = LanguageRegistry::build();
142        let src = "fn apply(lens: Lens) -> u32 {\n    lens.marks(total)\n}\n";
143        let idents = reg.symbols("a.rs", src).0;
144        let item = |line: usize, name: &str| {
145            idents
146                .iter()
147                .find(|ident| ident.line == line && ident.name == name)
148                .is_some_and(|ident| ident.item)
149        };
150        assert!(item(0, "apply"), "a definition");
151        assert!(item(0, "Lens"), "a type position");
152        assert!(item(1, "marks"), "a method call");
153        assert!(!item(1, "lens"), "the value it is called on");
154        assert!(!item(1, "total"), "an argument");
155    }
156
157    #[test]
158    fn a_file_without_a_grammar_falls_back_to_words() {
159        let reg = LanguageRegistry::build();
160        let idents = reg.symbols("notes.zzz", "total = price * 2qty\n").0;
161        assert_eq!(names_on(&idents, 0), ["total", "price"]);
162    }
163}
diffler_core/syntax/symbols.rs

diffler_core/syntax/
symbols.rs
