diffler_core/syntax/

symbols.rs

//! Every identifier in a file, by line and byte range.

use std::ops::Range;

use crate::syntax::registry::{LangEntry, LanguageRegistry};
use crate::syntax::scope::tag_pass;
use crate::syntax::{MAX_PARSE_BYTES, ScopeIndex, parse};

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Ident {
    pub line: usize,
    pub range: Range<usize>,
    pub name: String,
    /// A function, method or type: tagged by the grammar's tags query, or in
    /// a type position.
    pub item: bool,
}

impl LanguageRegistry {
    /// Every identifier in `content` and its definition spans, from one
    /// parse. Without a parse we fall back to plain words, strings and
    /// comments included.
    pub fn symbols(&self, path: &str, content: &str) -> (Vec<Ident>, ScopeIndex) {
        Self::symbols_as(self.for_file(path, content), content)
    }

    /// [`Self::symbols`] with `content` parsed as `entry`'s language.
    pub fn symbols_as(entry: Option<&LangEntry>, content: &str) -> (Vec<Ident>, ScopeIndex) {
        let entry = entry.filter(|_| content.len() <= MAX_PARSE_BYTES);
        let Some((entry, tree)) = entry.and_then(|entry| Some((entry, parse(entry, content)?)))
        else {
            return (words(content), ScopeIndex::default());
        };
        let (scope, tagged) = entry
            .tags()
            .map(|query| tag_pass(query, &tree, content))
            .unwrap_or_default();
        let mut idents = Vec::new();
        let mut cursor = tree.walk();
        loop {
            let node = cursor.node();
            if node.child_count() == 0 && !node.is_missing() && is_identifier(node.kind()) {
                let start = node.start_position();
                let len = node.end_byte() - node.start_byte();
                if let Some(name) = content
                    .get(node.start_byte()..node.end_byte())
                    .filter(|name| !name.is_empty())
                {
                    idents.push(Ident {
                        line: start.row,
                        range: start.column..start.column + len,
                        name: name.to_owned(),
                        item: node.kind().contains("type") || tagged.contains(&node.start_byte()),
                    });
                }
            }
            if cursor.goto_first_child() || cursor.goto_next_sibling() {
                continue;
            }
            loop {
                if !cursor.goto_parent() {
                    return (idents, scope);
                }
                if cursor.goto_next_sibling() {
                    break;
                }
            }
        }
    }
}

/// PHP spells an identifier `name` and Ruby `constant`.
fn is_identifier(kind: &str) -> bool {
    kind.ends_with("identifier") || kind == "name" || kind == "constant"
}

/// Every run of `[A-Za-z0-9_]` that does not start with a digit.
fn words(content: &str) -> Vec<Ident> {
    let mut out = Vec::new();
    for (line, text) in content.lines().enumerate() {
        let mut start = None;
        for (at, ch) in text
            .char_indices()
            .chain(std::iter::once((text.len(), ' ')))
        {
            let word_char = ch.is_ascii_alphanumeric() || ch == '_';
            match (start, word_char) {
                (None, true) => start = Some(at),
                (Some(from), false) => {
                    start = None;
                    let Some(token) = text.get(from..at) else {
                        continue;
                    };
                    if !token.starts_with(|first: char| first.is_ascii_digit()) {
                        out.push(Ident {
                            line,
                            range: from..at,
                            name: token.to_owned(),
                            item: false,
                        });
                    }
                }
                _ => {}
            }
        }
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;

    fn names_on(idents: &[Ident], line: usize) -> Vec<&str> {
        idents
            .iter()
            .filter(|ident| ident.line == line)
            .map(|ident| ident.name.as_str())
            .collect()
    }

    #[test]
    fn a_parsed_file_yields_its_names_and_skips_strings_and_comments() {
        let reg = LanguageRegistry::build();
        let src = "fn total(price: u32, qty: u32) -> u32 {\n    // price times qty\n    let label = \"qty\";\n    price * qty\n}\n";
        let idents = reg.symbols("a.rs", src).0;
        assert_eq!(
            names_on(&idents, 0),
            ["total", "price", "qty"],
            "the grammar parses a primitive type as its own node"
        );
        assert!(names_on(&idents, 1).is_empty(), "a comment names nothing");
        assert_eq!(names_on(&idents, 2), ["label"], "a string names nothing");
        assert_eq!(names_on(&idents, 3), ["price", "qty"]);
        let price = idents.iter().find(|i| i.line == 3).expect("price");
        assert_eq!(
            &src.lines().nth(3).expect("line")[price.range.clone()],
            "price"
        );
    }

    #[test]
    fn a_call_or_a_definition_names_an_item_and_a_value_does_not() {
        let reg = LanguageRegistry::build();
        let src = "fn apply(lens: Lens) -> u32 {\n    lens.marks(total)\n}\n";
        let idents = reg.symbols("a.rs", src).0;
        let item = |line: usize, name: &str| {
            idents
                .iter()
                .find(|ident| ident.line == line && ident.name == name)
                .is_some_and(|ident| ident.item)
        };
        assert!(item(0, "apply"), "a definition");
        assert!(item(0, "Lens"), "a type position");
        assert!(item(1, "marks"), "a method call");
        assert!(!item(1, "lens"), "the value it is called on");
        assert!(!item(1, "total"), "an argument");
    }

    #[test]
    fn a_file_without_a_grammar_falls_back_to_words() {
        let reg = LanguageRegistry::build();
        let idents = reg.symbols("notes.zzz", "total = price * 2qty\n").0;
        assert_eq!(names_on(&idents, 0), ["total", "price"]);
    }
}