ruff_python_trivia/
name_matcher.rs

1use memchr::memmem::Finder;
2use unicode_normalization::is_nfkc;
3
4/// A reusable text prefilter for an identifier or keyword.
5///
6/// A match only indicates that a source may contain the name: matches in
7/// comments and strings are included. Matchers created with [`Self::new`] accept
8/// sources that change under NFKC normalization because Python normalizes
9/// identifiers this way. Matchers created with [`Self::keyword`] search their literal
10/// spelling instead. Callers should validate candidates using the AST or semantic analysis.
11///
12/// Construct a matcher once and reuse it across sources to avoid preprocessing
13/// the name for each source. Cache matchers for fixed names in a [`std::sync::LazyLock`].
14#[derive(Debug, Clone)]
15pub struct NameMatcher<'a> {
16    finder: Finder<'a>,
17    is_keyword: bool,
18}
19
20impl<'a> NameMatcher<'a> {
21    /// Creates an identifier matcher that borrows `name` without allocating.
22    ///
23    /// The identifier `name` is expected to be NFKC-normalized.
24    pub fn new(name: &'a str) -> Self {
25        Self {
26            finder: Finder::new(name),
27            is_keyword: false,
28        }
29    }
30
31    /// Creates a keyword matcher that borrows `keyword` without allocating.
32    ///
33    /// Python recognizes keywords by their literal spelling, without NFKC normalization,
34    /// so keyword matchers skip the normalization check.
35    pub fn keyword(keyword: &'a str) -> Self {
36        Self {
37            finder: Finder::new(keyword),
38            is_keyword: true,
39        }
40    }
41
42    /// Returns whether `source` may contain the configured identifier or keyword.
43    ///
44    /// Identifier matchers conservatively return `true` if the source changes under
45    /// NFKC normalization. Otherwise, searches for the literal spelling bounded by bytes
46    /// other than ASCII letters, digits or `_`. Matches may occur in comments, strings,
47    /// or Unicode identifiers.
48    pub fn may_match(&self, source: &str) -> bool {
49        if !self.is_keyword && !source.is_ascii() && !is_nfkc(source) {
50            return true;
51        }
52
53        let bytes = source.as_bytes();
54        let len = self.finder.needle().len();
55        self.finder
56            .find_iter(bytes)
57            .any(|start| has_identifier_boundaries(bytes, start, start + len))
58    }
59}
60
61fn has_identifier_boundaries(source: &[u8], start: usize, end: usize) -> bool {
62    (start == 0 || !is_ascii_identifier_continue(source[start - 1]))
63        && source
64            .get(end)
65            .is_none_or(|byte| !is_ascii_identifier_continue(*byte))
66}
67
68fn is_ascii_identifier_continue(byte: u8) -> bool {
69    byte.is_ascii_alphanumeric() || byte == b'_'
70}
71
72#[cfg(test)]
73mod tests {
74    use super::NameMatcher;
75
76    #[test]
77    fn identifier_boundaries() {
78        let matcher = NameMatcher::new("x");
79        assert!(matcher.may_match("x"));
80        assert!(matcher.may_match("x = 1"));
81        assert!(matcher.may_match("obj.x"));
82        assert!(matcher.may_match("x()"));
83        assert!(matcher.may_match("exclude = x"));
84        assert!(matcher.may_match("xx + x"));
85
86        assert!(!matcher.may_match(""));
87        assert!(!matcher.may_match("exclude = 10"));
88        assert!(!matcher.may_match("ax"));
89        assert!(!matcher.may_match("x1"));
90        assert!(!matcher.may_match("1x"));
91        assert!(!matcher.may_match("_x"));
92        assert!(!matcher.may_match("x_"));
93    }
94
95    #[test]
96    fn strings_and_comments_are_candidates() {
97        let matcher = NameMatcher::new("name");
98        assert!(matcher.may_match(r#""name""#));
99        assert!(matcher.may_match("# name"));
100    }
101
102    #[test]
103    fn normalized_unicode_uses_literal_spelling() {
104        let matcher = NameMatcher::new("C");
105        assert!(!matcher.may_match("# note — a comment"));
106        assert!(!matcher.may_match("# café"));
107        assert!(!matcher.may_match("# 中文"));
108        assert!(!matcher.may_match("# שלום"));
109        assert!(!matcher.may_match("# 🦀"));
110        // There is no precomposed character for 'q' with an acute accent.
111        assert!(!matcher.may_match("# q\u{301}"));
112        assert!(matcher.may_match("# café\nC = 1"));
113        assert!(NameMatcher::new("café").may_match("café = 1"));
114    }
115
116    #[test]
117    fn normalization_can_change_identifiers() {
118        let matcher = NameMatcher::new("C");
119        assert!(matcher.may_match("𝒞 = 1"));
120        assert!(matcher.may_match("Ｃ = 1"));
121        assert!(NameMatcher::new("ffi").may_match("ﬃ = 1"));
122        assert!(NameMatcher::new("café").may_match("cafe\u{301} = 1"));
123    }
124
125    #[test]
126    fn keyword_uses_literal_spelling() {
127        let matcher = NameMatcher::keyword("class");
128        assert!(matcher.may_match("class C: pass"));
129        assert!(matcher.may_match("class Café: pass"));
130        assert!(matcher.may_match("# class"));
131        assert!(matcher.may_match(r#""class""#));
132        assert!(matcher.may_match("éclass = 1"));
133        assert!(!matcher.may_match("# café"));
134        assert!(!matcher.may_match("ｃｌａｓｓ = 1"));
135        assert!(!matcher.may_match("subclass = 1"));
136    }
137}
ruff_python_trivia/name_matcher.rs

ruff_python_trivia/
name_matcher.rs
