unicode_lang/
ident.rs

1//! Unicode identifier rules from [UAX #31], the `XID_Start` and `XID_Continue`
2//! derived properties.
3//!
4//! These are the properties a lexer consults to recognise identifiers in a
5//! programming language: the first scalar of an identifier must be
6//! `XID_Start`, and each subsequent scalar must be `XID_Continue`. The `XID`
7//! variants are the closure-stable forms — normalizing an identifier never
8//! turns a valid identifier into an invalid one — which is why they, rather
9//! than the older `ID_Start` / `ID_Continue`, are the recommended basis.
10//!
11//! [UAX #31]: https://www.unicode.org/reports/tr31/
12
13use crate::lookup::in_ranges;
14use crate::tables;
15
16/// Returns `true` if `c` may begin a Unicode identifier — that is, if `c` has
17/// the `XID_Start` property.
18///
19/// This is ASCII letters, plus the letters and letter-like marks of every
20/// script. It excludes digits, whitespace, and punctuation — including the
21/// underscore, which is `XID_Continue` but not `XID_Start`. A language that
22/// permits leading underscores (as Rust does) must allow that separately.
23///
24/// # Examples
25///
26/// ```
27/// use unicode_lang::is_xid_start;
28///
29/// assert!(is_xid_start('a'));
30/// assert!(is_xid_start('Δ'));      // Greek capital delta
31/// assert!(is_xid_start('日'));     // CJK ideograph
32///
33/// assert!(!is_xid_start('1'));     // a digit may continue, not start
34/// assert!(!is_xid_start('_'));     // underscore continues but does not start
35/// assert!(!is_xid_start(' '));
36/// ```
37#[inline]
38#[must_use]
39pub fn is_xid_start(c: char) -> bool {
40    in_ranges(c as u32, tables::XID_START)
41}
42
43/// Returns `true` if `c` may continue a Unicode identifier — that is, if `c`
44/// has the `XID_Continue` property.
45///
46/// `XID_Continue` is a superset of [`is_xid_start`]: it additionally admits
47/// decimal digits, the combining marks that attach to letters, and the Unicode
48/// connector punctuation — which includes the ASCII underscore `_` (category
49/// `Pc`). The underscore therefore continues an identifier but, per
50/// [`is_xid_start`], cannot begin one.
51///
52/// # Examples
53///
54/// ```
55/// use unicode_lang::is_xid_continue;
56///
57/// assert!(is_xid_continue('a'));
58/// assert!(is_xid_continue('9'));        // digits continue an identifier
59/// assert!(is_xid_continue('_'));        // connector punctuation continues
60///
61/// // A base letter followed by a combining mark is a valid continuation.
62/// assert!(is_xid_continue('\u{0301}')); // COMBINING ACUTE ACCENT
63/// ```
64#[inline]
65#[must_use]
66pub fn is_xid_continue(c: char) -> bool {
67    in_ranges(c as u32, tables::XID_CONTINUE)
68}
69
70/// Returns `true` if `s` is a well-formed Unicode identifier under the default
71/// [UAX #31] profile: it is non-empty, its first scalar is [`is_xid_start`],
72/// and every remaining scalar is [`is_xid_continue`].
73///
74/// This is a convenience over iterating the scalars by hand. It applies no
75/// language-specific extensions — notably it rejects a leading underscore, so
76/// callers whose grammar allows `_name` should test that case themselves.
77///
78/// [UAX #31]: https://www.unicode.org/reports/tr31/
79///
80/// # Examples
81///
82/// ```
83/// use unicode_lang::is_xid;
84///
85/// assert!(is_xid("total"));
86/// assert!(is_xid("Δpressure"));
87/// assert!(is_xid("café"));
88///
89/// assert!(!is_xid(""));         // empty is not an identifier
90/// assert!(!is_xid("1st"));      // cannot start with a digit
91/// assert!(!is_xid("a b"));      // no interior whitespace
92/// assert!(!is_xid("_name"));    // underscore is not XID; opt in separately
93/// ```
94#[must_use]
95pub fn is_xid(s: &str) -> bool {
96    let mut chars = s.chars();
97    match chars.next() {
98        Some(first) => is_xid_start(first) && chars.all(is_xid_continue),
99        None => false,
100    }
101}
102
103#[cfg(test)]
104mod tests {
105    use super::*;
106
107    #[test]
108    fn test_xid_start_ascii_matches_letters() {
109        for c in '\0'..='\u{7F}' {
110            let expected = c.is_ascii_alphabetic();
111            assert_eq!(is_xid_start(c), expected, "ascii {c:?}");
112        }
113    }
114
115    #[test]
116    fn test_xid_continue_ascii_matches_alnum_plus_underscore() {
117        // In ASCII, XID_Continue is exactly the alphanumerics plus the
118        // underscore (LOW LINE, connector punctuation).
119        for c in '\0'..='\u{7F}' {
120            let expected = c.is_ascii_alphanumeric() || c == '_';
121            assert_eq!(is_xid_continue(c), expected, "ascii {c:?}");
122        }
123    }
124
125    #[test]
126    fn test_xid_continue_superset_of_start() {
127        for cp in 0u32..=0x2FFF {
128            if let Some(c) = char::from_u32(cp) {
129                if is_xid_start(c) {
130                    assert!(is_xid_continue(c), "start but not continue: {c:?}");
131                }
132            }
133        }
134    }
135
136    #[test]
137    fn test_underscore_continues_but_does_not_start() {
138        assert!(!is_xid_start('_'));
139        assert!(is_xid_continue('_'));
140        assert!(!is_xid("_name")); // leading underscore fails XID_Start
141    }
142
143    #[test]
144    fn test_is_xid_empty_returns_false() {
145        assert!(!is_xid(""));
146    }
147
148    #[test]
149    fn test_is_xid_rejects_digit_start() {
150        assert!(!is_xid("1st"));
151    }
152
153    #[test]
154    fn test_is_xid_accepts_unicode() {
155        assert!(is_xid("café"));
156        assert!(is_xid("Δx"));
157        assert!(is_xid("naïve"));
158    }
159}
unicode_lang/ident.rs

unicode_lang/
ident.rs
