libxml_rs/xml/xpath/
lexer.rs

1//! XPath 1.0 Expression Lexer/Tokenizer (§25).
2//!
3//! Tokenizes XPath expression strings into a stream of tokens
4//! for the parser to consume.
5//!
6//! # UPSTREAM-PARITY
7//!
8//! Covers all XPath 1.0 token types: names, numbers, strings, operators,
9//! axes, function names, variable references, punctuation.
10//!
11//! # Courts
12//!
13//! XPATH-LEXER-*
14//!
15//! # Upstream contract
16//!
17//! Mirrors the tokenizer half of upstream `xpath.c`
18//! (`SRC-LIBXML2-2.15.0-XPATH-C`, parity target libxml2 2.15.3 oracle):
19//! xmlXPathLexer token classes (names, numbers, strings, operators, axes,
20//! function names, variable references, punctuation) over the XPath 1.0
21//! grammar.
22//!
23//! # Conceptual behavior
24//!
25//! Implements a hand-written scanner: names (NCName/QName), `*` wildcard,
26//! `.`/`..` abbreviations, `@`, `::`, `/` and `//`, `|`, `+`/`-`, the six
27//! comparison operators, string literals, numbers, and the `$var`
28//! reference form — with the upstream maximal-munch behavior for
29//! ambiguous sequences.
30//!
31//! # Ownership & safety invariants
32//!
33//! Tokens own their payloads as Rust Strings — no pointers into the
34//! expression buffer escape the token stream, so a lexer result is fully
35//! owned and safe to store. This differs from upstream, where tokens are
36//! offsets into the caller expression string.
37//!
38//! # Historical quirks & epochs
39//!
40//! The token model tracks the upstream lexer as of the 2.15.3 oracle;
41//! R-000105 (node tests vs function calls) is resolved at the parser
42//! layer but depends on the lexer not swallowing `(` after node-test
43//! names.
44//!
45//! # Deliberate oddities
46//!
47//! Numbers are tokenized as raw text and converted later by the
48//! R-000166 string-eval-number port, reproducing the oracle digit
49//! accumulation (MAX_FRAC=20 cap, exponent underflow) instead of Rust
50//! f64::from_str, which accepts forms the oracle rejects (e.g. leading
51//! `+`).
52//!
53//! # Proving courts
54//!
55//! XPATH-LEXER-* and the XPATH differential probes compile expressions
56//! byte-identical against the oracle; cargo test covers the tokenizer
57//! unit suites.
58//!
59//! # Tempting simplifications that would break parity
60//!
61//! Do not convert number literals with Rust float parsing: the oracle
62//! accumulation differs on leading `+`, exponent limits and underflow
63//! (R-000166). Do not return borrowed spans into the expression string:
64//! owned tokens keep the AST independent of the source text lifetime.
65
66use std::fmt;
67
68// ═══════════════════════════════════════════════════════════════════════════════
69// Token Types
70// ═══════════════════════════════════════════════════════════════════════════════
71
72/// A token in an XPath expression.
73#[derive(Debug, Clone, PartialEq)]
74pub enum Token {
75    // ── Names ────────────────────────────────────────────────────────────
76    /// Name (NCName or QName)
77    Name(String),
78    /// `*` wildcard
79    Star,
80    /// `.` (self)
81    Dot,
82    /// `..` (parent)
83    DotDot,
84
85    // ── Operators ────────────────────────────────────────────────────────
86    /// `@` (attribute axis)
87    At,
88    /// `::` (axis separator)
89    DoubleColon,
90    /// `/`
91    Slash,
92    /// `//`
93    DoubleSlash,
94    /// `|`
95    Pipe,
96    /// `+`
97    Plus,
98    /// `-`
99    Minus,
100    /// `=`
101    Eq,
102    /// `!=`
103    Ne,
104    /// `<`
105    Lt,
106    /// `>`
107    Gt,
108    /// `<=`
109    Le,
110    /// `>=`
111    Ge,
112    /// `*` (multiplication operator, distinct from wildcard)
113    Multiply,
114
115    // ── Keywords ─────────────────────────────────────────────────────────
116    /// `or`
117    Or,
118    /// `and`
119    And,
120    /// `mod`
121    Mod,
122    /// `div`
123    Div,
124    /// `ancestor`
125    Ancestor,
126    /// `ancestor-or-self`
127    AncestorOrSelf,
128    /// `attribute`
129    Attribute,
130    /// `child`
131    Child,
132    /// `descendant`
133    Descendant,
134    /// `descendant-or-self`
135    DescendantOrSelf,
136    /// `following`
137    Following,
138    /// `following-sibling`
139    FollowingSibling,
140    /// `namespace`
141    Namespace,
142    /// `parent`
143    Parent,
144    /// `preceding`
145    Preceding,
146    /// `preceding-sibling`
147    PrecedingSibling,
148    /// `self`
149    Self_,
150
151    // ── Literals ─────────────────────────────────────────────────────────
152    /// String literal (without quotes)
153    StringLiteral(String),
154    /// Numeric literal
155    NumberLiteral(f64),
156
157    // ── Punctuation ──────────────────────────────────────────────────────
158    /// `(` — left parenthesis (groups sub-expressions, opens function calls)
159    LParen,
160    /// `)` — right parenthesis
161    RParen,
162    /// `[` — left bracket (opens a predicate)
163    LBracket,
164    /// `]` — right bracket (closes a predicate)
165    RBracket,
166    /// `{` — left brace (for XSLT attribute value templates; rare in XPath)
167    LBrace,
168    /// `}` — right brace
169    RBrace,
170    /// `,` — separates function call arguments
171    Comma,
172    /// `$` (variable reference)
173    Dollar,
174
175    // ── Special ──────────────────────────────────────────────────────────
176    /// End of expression
177    Eof,
178}
179
180impl fmt::Display for Token {
181    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
182        match self {
183            Token::Name(n) => write!(f, "{}", n),
184            Token::Star => write!(f, "*"),
185            Token::Dot => write!(f, "."),
186            Token::DotDot => write!(f, ".."),
187            Token::At => write!(f, "@"),
188            Token::DoubleColon => write!(f, "::"),
189            Token::Slash => write!(f, "/"),
190            Token::DoubleSlash => write!(f, "//"),
191            Token::Pipe => write!(f, "|"),
192            Token::Plus => write!(f, "+"),
193            Token::Minus => write!(f, "-"),
194            Token::Eq => write!(f, "="),
195            Token::Ne => write!(f, "!="),
196            Token::Lt => write!(f, "<"),
197            Token::Gt => write!(f, ">"),
198            Token::Le => write!(f, "<="),
199            Token::Ge => write!(f, ">="),
200            Token::Multiply => write!(f, "*"),
201            Token::Or => write!(f, "or"),
202            Token::And => write!(f, "and"),
203            Token::Mod => write!(f, "mod"),
204            Token::Div => write!(f, "div"),
205            Token::Ancestor => write!(f, "ancestor"),
206            Token::AncestorOrSelf => write!(f, "ancestor-or-self"),
207            Token::Attribute => write!(f, "attribute"),
208            Token::Child => write!(f, "child"),
209            Token::Descendant => write!(f, "descendant"),
210            Token::DescendantOrSelf => write!(f, "descendant-or-self"),
211            Token::Following => write!(f, "following"),
212            Token::FollowingSibling => write!(f, "following-sibling"),
213            Token::Namespace => write!(f, "namespace"),
214            Token::Parent => write!(f, "parent"),
215            Token::Preceding => write!(f, "preceding"),
216            Token::PrecedingSibling => write!(f, "preceding-sibling"),
217            Token::Self_ => write!(f, "self"),
218            Token::StringLiteral(s) => write!(f, "'{}'", s),
219            Token::NumberLiteral(n) => write!(f, "{}", n),
220            Token::LParen => write!(f, "("),
221            Token::RParen => write!(f, ")"),
222            Token::LBracket => write!(f, "["),
223            Token::RBracket => write!(f, "]"),
224            Token::LBrace => write!(f, "{{"),
225            Token::RBrace => write!(f, "}}"),
226            Token::Comma => write!(f, ","),
227            Token::Dollar => write!(f, "$"),
228            Token::Eof => write!(f, "<EOF>"),
229        }
230    }
231}
232
233// ═══════════════════════════════════════════════════════════════════════════════
234// Lexer
235// ═══════════════════════════════════════════════════════════════════════════════
236
237/// XPath expression lexer.
238///
239/// Produces a stream of tokens from an XPath expression string.
240#[derive(Debug, Clone)]
241pub struct Lexer {
242    /// Input bytes
243    input: Vec<u8>,
244    /// Current position
245    pos: usize,
246    /// Look-ahead character (0 if EOF)
247    ch: u8,
248    /// Whether we're at the start of an expression (helps with `-` vs `-`)
249    at_start: bool,
250}
251
252impl Lexer {
253    /// Create a lexer for the given XPath expression string.
254    pub fn new(input: &str) -> Self {
255        let bytes = input.as_bytes().to_vec();
256        let ch = if bytes.is_empty() { 0 } else { bytes[0] };
257        Self {
258            input: bytes,
259            pos: 0,
260            ch,
261            at_start: true,
262        }
263    }
264
265    /// Advance to the next character.
266    fn advance(&mut self) {
267        self.pos += 1;
268        self.ch = if self.pos < self.input.len() {
269            self.input[self.pos]
270        } else {
271            0
272        };
273    }
274
275    /// Peek at the next character without consuming it.
276    fn peek(&self) -> u8 {
277        if self.pos + 1 < self.input.len() {
278            self.input[self.pos + 1]
279        } else {
280            0
281        }
282    }
283
284    /// Skip whitespace.
285    fn skip_ws(&mut self) {
286        while self.ch != 0
287            && (self.ch == b' ' || self.ch == b'\t' || self.ch == b'\n' || self.ch == b'\r')
288        {
289            self.advance();
290        }
291    }
292
293    /// Read a name token (NCName).
294    fn read_name(&mut self) -> String {
295        let start = self.pos;
296        while self.ch != 0
297            && (self.ch.is_ascii_alphanumeric()
298                || self.ch == b'_'
299                || self.ch == b'-'
300                || self.ch == b'.')
301        {
302            self.advance();
303        }
304        String::from_utf8_lossy(&self.input[start..self.pos]).to_string()
305    }
306
307    /// Try to match an axis name or keyword.
308    fn try_keyword_or_axis(&self, name: &str) -> Option<Token> {
309        match name {
310            "or" => Some(Token::Or),
311            "and" => Some(Token::And),
312            "mod" => Some(Token::Mod),
313            "div" => Some(Token::Div),
314            "ancestor" => Some(Token::Ancestor),
315            "ancestor-or-self" => Some(Token::AncestorOrSelf),
316            "attribute" => Some(Token::Attribute),
317            "child" => Some(Token::Child),
318            "descendant" => Some(Token::Descendant),
319            "descendant-or-self" => Some(Token::DescendantOrSelf),
320            "following" => Some(Token::Following),
321            "following-sibling" => Some(Token::FollowingSibling),
322            "namespace" => Some(Token::Namespace),
323            "parent" => Some(Token::Parent),
324            "preceding" => Some(Token::Preceding),
325            "preceding-sibling" => Some(Token::PrecedingSibling),
326            "self" => Some(Token::Self_),
327            _ => None,
328        }
329    }
330
331    /// Read a numeric literal — a faithful port of upstream xpath.c
332    /// `xmlXPathCompNumber` (R-000166). The oracle accumulates digits
333    /// directly (`ret = ret * 10 + d`), caps the fraction at MAX_FRAC=20
334    /// digits after any leading zeros, and applies the exponent with
335    /// `pow(10.0, exp)` — which underflows to 0 for exponents below the
336    /// smallest subnormal (e.g. `5e-324`). Rust's correctly-rounded
337    /// `strtod`-style parse differs in those edge cases, so the accumulation
338    /// is reproduced exactly.
339    fn read_number(&mut self) -> f64 {
340        let input = &self.input;
341        let len = input.len();
342        let mut cur = self.pos;
343
344        // Integer part.
345        let mut ret = 0.0f64;
346        while cur < len && input[cur].is_ascii_digit() {
347            ret = ret * 10.0 + (input[cur] - b'0') as f64;
348            cur += 1;
349        }
350
351        // Fractional part (upstream consumes a trailing '.' even without
352        // digits, so `5.` is a single number literal).
353        let mut frac: i32 = 0;
354        if cur < len && input[cur] == b'.' {
355            cur += 1;
356            while cur < len && input[cur] == b'0' {
357                frac += 1;
358                cur += 1;
359            }
360            let max = frac + 20; // MAX_FRAC
361            let mut fraction = 0.0f64;
362            while cur < len && input[cur].is_ascii_digit() && frac < max {
363                let v = (input[cur] - b'0') as f64;
364                fraction = fraction * 10.0 + v;
365                frac += 1;
366                cur += 1;
367            }
368            fraction /= 10f64.powf(frac as f64);
369            ret += fraction;
370            while cur < len && input[cur].is_ascii_digit() {
371                cur += 1;
372            }
373        }
374
375        // Exponent part (upstream xmlXPathCompNumber consumes 'e'/'E'
376        // unconditionally, then an optional sign, then digits — greedily
377        // even when malformed).
378        let mut exponent: i32 = 0;
379        let mut is_exponent_negative = false;
380        if cur < len && (input[cur] == b'e' || input[cur] == b'E') {
381            cur += 1;
382            if cur < len && input[cur] == b'-' {
383                is_exponent_negative = true;
384                cur += 1;
385            } else if cur < len && input[cur] == b'+' {
386                cur += 1;
387            }
388            while cur < len && input[cur].is_ascii_digit() {
389                if exponent < 1000000 {
390                    exponent = exponent * 10 + (input[cur] - b'0') as i32;
391                }
392                cur += 1;
393            }
394        }
395        if is_exponent_negative {
396            exponent = -exponent;
397        }
398        ret *= 10f64.powf(exponent as f64);
399
400        self.pos = cur;
401        self.ch = if cur < len { input[cur] } else { 0 };
402        ret
403    }
404
405    /// Read a string literal.
406    fn read_string(&mut self, quote: u8) -> String {
407        self.advance(); // consume opening quote
408        let start = self.pos;
409        while self.ch != 0 && self.ch != quote {
410            self.advance();
411        }
412        let s = String::from_utf8_lossy(&self.input[start..self.pos]).to_string();
413        if self.ch == quote {
414            self.advance(); // consume closing quote
415        }
416        s
417    }
418
419    /// Get the next token.
420    pub fn next_token(&mut self) -> Token {
421        self.skip_ws();
422
423        if self.ch == 0 {
424            return Token::Eof;
425        }
426
427        // Save at_start for unary minus detection
428        let _was_at_start = self.at_start;
429        self.at_start = false;
430
431        // ── Single-char tokens ────────────────────────────────────────────
432        match self.ch {
433            b'(' => {
434                self.advance();
435                return Token::LParen;
436            }
437            b')' => {
438                self.advance();
439                return Token::RParen;
440            }
441            b'[' => {
442                self.advance();
443                return Token::LBracket;
444            }
445            b']' => {
446                self.advance();
447                return Token::RBracket;
448            }
449            b'{' => {
450                self.advance();
451                return Token::LBrace;
452            }
453            b'}' => {
454                self.advance();
455                return Token::RBrace;
456            }
457            b',' => {
458                self.advance();
459                return Token::Comma;
460            }
461            b'$' => {
462                self.advance();
463                return Token::Dollar;
464            }
465            b'|' => {
466                self.advance();
467                return Token::Pipe;
468            }
469            b'+' => {
470                self.advance();
471                return Token::Plus;
472            }
473            b'@' => {
474                self.advance();
475                return Token::At;
476            }
477            b'.' => {
478                if self.peek() == b'.' {
479                    self.advance();
480                    self.advance();
481                    return Token::DotDot;
482                }
483                // Check if it's a number starting with '.'
484                if self.peek().is_ascii_digit() {
485                    return Token::NumberLiteral(self.read_number());
486                }
487                self.advance();
488                return Token::Dot;
489            }
490            b'-' => {
491                self.advance();
492                // If at start or after operator, this is unary minus
493                // We handle this at the parser level, just return Minus
494                return Token::Minus;
495            }
496            b'=' => {
497                self.advance();
498                return Token::Eq;
499            }
500            b'!' => {
501                if self.peek() == b'=' {
502                    self.advance();
503                    self.advance();
504                    return Token::Ne;
505                }
506                // Invalid character, skip
507                self.advance();
508                return self.next_token();
509            }
510            b'<' => {
511                self.advance();
512                if self.ch == b'=' {
513                    self.advance();
514                    return Token::Le;
515                }
516                return Token::Lt;
517            }
518            b'>' => {
519                self.advance();
520                if self.ch == b'=' {
521                    self.advance();
522                    return Token::Ge;
523                }
524                return Token::Gt;
525            }
526            b'/' => {
527                self.advance();
528                if self.ch == b'/' {
529                    self.advance();
530                    return Token::DoubleSlash;
531                }
532                return Token::Slash;
533            }
534            b'*' => {
535                self.advance();
536                return Token::Star; // lexer returns Star; parser disambiguates
537            }
538            b':' => {
539                if self.peek() == b':' {
540                    self.advance();
541                    self.advance();
542                    return Token::DoubleColon;
543                }
544                // Single colon is part of a QName, handled below
545                // Actually, if we see a colon, it should be part of a name
546                // This case handles axis::name or prefix:name
547                // Since we read the full name first, this shouldn't normally happen alone
548                self.advance();
549                return self.next_token();
550            }
551            b'\'' | b'"' => {
552                let quote = self.ch;
553                let s = self.read_string(quote);
554                return Token::StringLiteral(s);
555            }
556            _ => {}
557        }
558
559        // ── Number ───────────────────────────────────────────────────────
560        if self.ch.is_ascii_digit() {
561            return Token::NumberLiteral(self.read_number());
562        }
563
564        // ── Name ─────────────────────────────────────────────────────────
565        if self.ch.is_ascii_alphabetic() || self.ch == b'_' {
566            let name = self.read_name();
567
568            // Check for QName (prefix:local)
569            if self.ch == b':' && self.peek() != b':' {
570                self.advance(); // consume ':'
571                if self.ch.is_ascii_alphabetic() || self.ch == b'_' || self.ch == b'*' {
572                    if self.ch == b'*' {
573                        self.advance();
574                        let full = format!("{}:*", name);
575                        return Token::Name(full);
576                    }
577                    let local = self.read_name();
578                    return Token::Name(format!("{}:{}", name, local));
579                }
580                // If the colon is not followed by a valid name character,
581                // it might be an axis separator that got split. Push back?
582                // Actually in well-formed XPath, `name:` is followed by `:`
583                // for axis:: or by a local name for QName.
584                // We already checked peek != ':', so this is a QName prefix.
585                // If the local part is missing, treat the whole thing as a name.
586                return Token::Name(name);
587            }
588
589            // Check for axis separator: name::
590            // We DON'T consume the :: here — we return just the axis keyword token.
591            // The :: will be tokenized as DoubleColon on the next call to next_token().
592            if self.ch == b':' && self.peek() == b':' {
593                if let Some(axis) = self.try_keyword_or_axis(&name) {
594                    return axis;
595                }
596                // Not an axis keyword — could be a QName prefix followed by ::?
597                // Treat it as a regular name and let the :: be consumed separately.
598                return Token::Name(name);
599            }
600
601            // Check for keyword or axis
602            if let Some(keyword) = self.try_keyword_or_axis(&name) {
603                return keyword;
604            }
605
606            return Token::Name(name);
607        }
608
609        // Unknown character, skip
610        self.advance();
611        self.next_token()
612    }
613}
614
615// ═══════════════════════════════════════════════════════════════════════════════
616// Tests
617// ═══════════════════════════════════════════════════════════════════════════════
618
619#[cfg(test)]
620mod tests {
621    use super::*;
622
623    fn tokenize(s: &str) -> Vec<Token> {
624        let mut lexer = Lexer::new(s);
625        let mut tokens = Vec::new();
626        loop {
627            let tok = lexer.next_token();
628            let is_eof = matches!(tok, Token::Eof);
629            tokens.push(tok);
630            if is_eof {
631                break;
632            }
633        }
634        tokens
635    }
636
637    #[test]
638    fn test_empty() {
639        let tokens = tokenize("");
640        assert_eq!(tokens.len(), 1);
641        assert_eq!(tokens[0], Token::Eof);
642    }
643
644    #[test]
645    fn test_simple_path() {
646        let tokens = tokenize("child::para");
647        assert_eq!(
648            tokens,
649            vec![
650                Token::Child,
651                Token::DoubleColon,
652                Token::Name("para".into()),
653                Token::Eof,
654            ]
655        );
656    }
657
658    #[test]
659    fn test_absolute_path() {
660        let tokens = tokenize("/child::para");
661        assert_eq!(
662            tokens,
663            vec![
664                Token::Slash,
665                Token::Child,
666                Token::DoubleColon,
667                Token::Name("para".into()),
668                Token::Eof,
669            ]
670        );
671    }
672
673    #[test]
674    fn test_short_form() {
675        let tokens = tokenize("para");
676        assert_eq!(tokens, vec![Token::Name("para".into()), Token::Eof]);
677    }
678
679    #[test]
680    fn test_attribute() {
681        let tokens = tokenize("@attr");
682        assert_eq!(
683            tokens,
684            vec![Token::At, Token::Name("attr".into()), Token::Eof]
685        );
686    }
687
688    #[test]
689    fn test_predicate() {
690        let tokens = tokenize("para[1]");
691        assert_eq!(
692            tokens,
693            vec![
694                Token::Name("para".into()),
695                Token::LBracket,
696                Token::NumberLiteral(1.0),
697                Token::RBracket,
698                Token::Eof,
699            ]
700        );
701    }
702
703    #[test]
704    fn test_function_call() {
705        let tokens = tokenize("position()");
706        assert_eq!(
707            tokens,
708            vec![
709                Token::Name("position".into()),
710                Token::LParen,
711                Token::RParen,
712                Token::Eof,
713            ]
714        );
715    }
716
717    #[test]
718    fn test_string_literal() {
719        let tokens = tokenize("'hello'");
720        assert_eq!(
721            tokens,
722            vec![Token::StringLiteral("hello".into()), Token::Eof]
723        );
724    }
725
726    #[test]
727    fn test_number() {
728        let tokens = tokenize("42");
729        assert_eq!(tokens, vec![Token::NumberLiteral(42.0), Token::Eof]);
730    }
731    #[allow(clippy::approx_constant)]
732    #[test]
733    fn test_decimal() {
734        let tokens = tokenize("3.14");
735        assert_eq!(tokens, vec![Token::NumberLiteral(3.14), Token::Eof]);
736    }
737
738    #[test]
739    fn test_operators() {
740        let tokens = tokenize("a = b and c != d or e < f");
741        assert!(tokens.contains(&Token::Eq));
742        assert!(tokens.contains(&Token::And));
743        assert!(tokens.contains(&Token::Ne));
744        assert!(tokens.contains(&Token::Or));
745        assert!(tokens.contains(&Token::Lt));
746    }
747
748    #[test]
749    fn test_union() {
750        let tokens = tokenize("a | b");
751        assert_eq!(
752            tokens,
753            vec![
754                Token::Name("a".into()),
755                Token::Pipe,
756                Token::Name("b".into()),
757                Token::Eof,
758            ]
759        );
760    }
761
762    #[test]
763    fn test_double_slash() {
764        let tokens = tokenize("//para");
765        assert_eq!(
766            tokens,
767            vec![Token::DoubleSlash, Token::Name("para".into()), Token::Eof]
768        );
769    }
770
771    #[test]
772    fn test_qname() {
773        let tokens = tokenize("xslt:template");
774        assert_eq!(
775            tokens,
776            vec![Token::Name("xslt:template".into()), Token::Eof]
777        );
778    }
779
780    #[test]
781    fn test_wildcard() {
782        let tokens = tokenize("*");
783        assert_eq!(tokens, vec![Token::Star, Token::Eof]);
784    }
785
786    #[test]
787    fn test_ns_wildcard() {
788        let tokens = tokenize("ns:*");
789        assert_eq!(tokens, vec![Token::Name("ns:*".into()), Token::Eof]);
790    }
791
792    #[test]
793    fn test_dot_dot() {
794        let tokens = tokenize("..");
795        assert_eq!(tokens, vec![Token::DotDot, Token::Eof]);
796    }
797
798    #[test]
799    fn test_axis_keyword() {
800        let tokens = tokenize("ancestor-or-self::node()");
801        assert_eq!(
802            tokens,
803            vec![
804                Token::AncestorOrSelf,
805                Token::DoubleColon,
806                Token::Name("node".into()),
807                Token::LParen,
808                Token::RParen,
809                Token::Eof,
810            ]
811        );
812    }
813
814    #[test]
815    fn test_complex_expression() {
816        let tokens = tokenize("/html/body//div[@class='main']/p[1]");
817        // Collect name-like tokens (including keyword tokens that can be element names)
818        let names: Vec<String> = tokens
819            .iter()
820            .filter_map(|t| match t {
821                Token::Name(n) => Some(n.clone()),
822                Token::Div => Some("div".to_string()),
823                Token::Mod => Some("mod".to_string()),
824                Token::And => Some("and".to_string()),
825                Token::Or => Some("or".to_string()),
826                _ => None,
827            })
828            .collect();
829        assert_eq!(names, vec!["html", "body", "div", "class", "p"]);
830    }
831
832    #[test]
833    fn test_variable() {
834        let tokens = tokenize("$var");
835        assert_eq!(
836            tokens,
837            vec![Token::Dollar, Token::Name("var".into()), Token::Eof]
838        );
839    }
840}
libxml_rs/xml/xpath/lexer.rs

libxml_rs/xml/xpath/
lexer.rs
