libxml_rs/xml/xpath/lexer.rs
1//! XPath 1.0 Expression Lexer/Tokenizer (§25).
2//!
3//! Tokenizes XPath expression strings into a stream of tokens
4//! for the parser to consume.
5//!
6//! # UPSTREAM-PARITY
7//!
8//! Covers all XPath 1.0 token types: names, numbers, strings, operators,
9//! axes, function names, variable references, punctuation.
10//!
11//! # Courts
12//!
13//! XPATH-LEXER-*
14//!
15//! # Upstream contract
16//!
17//! Mirrors the tokenizer half of upstream `xpath.c`
18//! (`SRC-LIBXML2-2.15.0-XPATH-C`, parity target libxml2 2.15.3 oracle):
19//! xmlXPathLexer token classes (names, numbers, strings, operators, axes,
20//! function names, variable references, punctuation) over the XPath 1.0
21//! grammar.
22//!
23//! # Conceptual behavior
24//!
25//! Implements a hand-written scanner: names (NCName/QName), `*` wildcard,
26//! `.`/`..` abbreviations, `@`, `::`, `/` and `//`, `|`, `+`/`-`, the six
27//! comparison operators, string literals, numbers, and the `$var`
28//! reference form — with the upstream maximal-munch behavior for
29//! ambiguous sequences.
30//!
31//! # Ownership & safety invariants
32//!
33//! Tokens own their payloads as Rust Strings — no pointers into the
34//! expression buffer escape the token stream, so a lexer result is fully
35//! owned and safe to store. This differs from upstream, where tokens are
36//! offsets into the caller expression string.
37//!
38//! # Historical quirks & epochs
39//!
40//! The token model tracks the upstream lexer as of the 2.15.3 oracle;
41//! R-000105 (node tests vs function calls) is resolved at the parser
42//! layer but depends on the lexer not swallowing `(` after node-test
43//! names.
44//!
45//! # Deliberate oddities
46//!
47//! Numbers are tokenized as raw text and converted later by the
48//! R-000166 string-eval-number port, reproducing the oracle digit
49//! accumulation (MAX_FRAC=20 cap, exponent underflow) instead of Rust
50//! f64::from_str, which accepts forms the oracle rejects (e.g. leading
51//! `+`).
52//!
53//! # Proving courts
54//!
55//! XPATH-LEXER-* and the XPATH differential probes compile expressions
56//! byte-identical against the oracle; cargo test covers the tokenizer
57//! unit suites.
58//!
59//! # Tempting simplifications that would break parity
60//!
61//! Do not convert number literals with Rust float parsing: the oracle
62//! accumulation differs on leading `+`, exponent limits and underflow
63//! (R-000166). Do not return borrowed spans into the expression string:
64//! owned tokens keep the AST independent of the source text lifetime.
65
66use std::fmt;
67
68// ═══════════════════════════════════════════════════════════════════════════════
69// Token Types
70// ═══════════════════════════════════════════════════════════════════════════════
71
72/// A token in an XPath expression.
73#[derive(Debug, Clone, PartialEq)]
74pub enum Token {
75 // ── Names ────────────────────────────────────────────────────────────
76 /// Name (NCName or QName)
77 Name(String),
78 /// `*` wildcard
79 Star,
80 /// `.` (self)
81 Dot,
82 /// `..` (parent)
83 DotDot,
84
85 // ── Operators ────────────────────────────────────────────────────────
86 /// `@` (attribute axis)
87 At,
88 /// `::` (axis separator)
89 DoubleColon,
90 /// `/`
91 Slash,
92 /// `//`
93 DoubleSlash,
94 /// `|`
95 Pipe,
96 /// `+`
97 Plus,
98 /// `-`
99 Minus,
100 /// `=`
101 Eq,
102 /// `!=`
103 Ne,
104 /// `<`
105 Lt,
106 /// `>`
107 Gt,
108 /// `<=`
109 Le,
110 /// `>=`
111 Ge,
112 /// `*` (multiplication operator, distinct from wildcard)
113 Multiply,
114
115 // ── Keywords ─────────────────────────────────────────────────────────
116 /// `or`
117 Or,
118 /// `and`
119 And,
120 /// `mod`
121 Mod,
122 /// `div`
123 Div,
124 /// `ancestor`
125 Ancestor,
126 /// `ancestor-or-self`
127 AncestorOrSelf,
128 /// `attribute`
129 Attribute,
130 /// `child`
131 Child,
132 /// `descendant`
133 Descendant,
134 /// `descendant-or-self`
135 DescendantOrSelf,
136 /// `following`
137 Following,
138 /// `following-sibling`
139 FollowingSibling,
140 /// `namespace`
141 Namespace,
142 /// `parent`
143 Parent,
144 /// `preceding`
145 Preceding,
146 /// `preceding-sibling`
147 PrecedingSibling,
148 /// `self`
149 Self_,
150
151 // ── Literals ─────────────────────────────────────────────────────────
152 /// String literal (without quotes)
153 StringLiteral(String),
154 /// Numeric literal
155 NumberLiteral(f64),
156
157 // ── Punctuation ──────────────────────────────────────────────────────
158 /// `(` — left parenthesis (groups sub-expressions, opens function calls)
159 LParen,
160 /// `)` — right parenthesis
161 RParen,
162 /// `[` — left bracket (opens a predicate)
163 LBracket,
164 /// `]` — right bracket (closes a predicate)
165 RBracket,
166 /// `{` — left brace (for XSLT attribute value templates; rare in XPath)
167 LBrace,
168 /// `}` — right brace
169 RBrace,
170 /// `,` — separates function call arguments
171 Comma,
172 /// `$` (variable reference)
173 Dollar,
174
175 // ── Special ──────────────────────────────────────────────────────────
176 /// End of expression
177 Eof,
178}
179
180impl fmt::Display for Token {
181 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
182 match self {
183 Token::Name(n) => write!(f, "{}", n),
184 Token::Star => write!(f, "*"),
185 Token::Dot => write!(f, "."),
186 Token::DotDot => write!(f, ".."),
187 Token::At => write!(f, "@"),
188 Token::DoubleColon => write!(f, "::"),
189 Token::Slash => write!(f, "/"),
190 Token::DoubleSlash => write!(f, "//"),
191 Token::Pipe => write!(f, "|"),
192 Token::Plus => write!(f, "+"),
193 Token::Minus => write!(f, "-"),
194 Token::Eq => write!(f, "="),
195 Token::Ne => write!(f, "!="),
196 Token::Lt => write!(f, "<"),
197 Token::Gt => write!(f, ">"),
198 Token::Le => write!(f, "<="),
199 Token::Ge => write!(f, ">="),
200 Token::Multiply => write!(f, "*"),
201 Token::Or => write!(f, "or"),
202 Token::And => write!(f, "and"),
203 Token::Mod => write!(f, "mod"),
204 Token::Div => write!(f, "div"),
205 Token::Ancestor => write!(f, "ancestor"),
206 Token::AncestorOrSelf => write!(f, "ancestor-or-self"),
207 Token::Attribute => write!(f, "attribute"),
208 Token::Child => write!(f, "child"),
209 Token::Descendant => write!(f, "descendant"),
210 Token::DescendantOrSelf => write!(f, "descendant-or-self"),
211 Token::Following => write!(f, "following"),
212 Token::FollowingSibling => write!(f, "following-sibling"),
213 Token::Namespace => write!(f, "namespace"),
214 Token::Parent => write!(f, "parent"),
215 Token::Preceding => write!(f, "preceding"),
216 Token::PrecedingSibling => write!(f, "preceding-sibling"),
217 Token::Self_ => write!(f, "self"),
218 Token::StringLiteral(s) => write!(f, "'{}'", s),
219 Token::NumberLiteral(n) => write!(f, "{}", n),
220 Token::LParen => write!(f, "("),
221 Token::RParen => write!(f, ")"),
222 Token::LBracket => write!(f, "["),
223 Token::RBracket => write!(f, "]"),
224 Token::LBrace => write!(f, "{{"),
225 Token::RBrace => write!(f, "}}"),
226 Token::Comma => write!(f, ","),
227 Token::Dollar => write!(f, "$"),
228 Token::Eof => write!(f, "<EOF>"),
229 }
230 }
231}
232
233// ═══════════════════════════════════════════════════════════════════════════════
234// Lexer
235// ═══════════════════════════════════════════════════════════════════════════════
236
237/// XPath expression lexer.
238///
239/// Produces a stream of tokens from an XPath expression string.
240#[derive(Debug, Clone)]
241pub struct Lexer {
242 /// Input bytes
243 input: Vec<u8>,
244 /// Current position
245 pos: usize,
246 /// Look-ahead character (0 if EOF)
247 ch: u8,
248 /// Whether we're at the start of an expression (helps with `-` vs `-`)
249 at_start: bool,
250 /// Byte offset at which each produced token starts (parallel to the
251 /// caller's token stream; feeds upstream's XPath error caret).
252 token_starts: Vec<usize>,
253}
254
255impl Lexer {
256 /// Create a lexer for the given XPath expression string.
257 pub fn new(input: &str) -> Self {
258 let bytes = input.as_bytes().to_vec();
259 let ch = if bytes.is_empty() { 0 } else { bytes[0] };
260 Self {
261 input: bytes,
262 pos: 0,
263 ch,
264 at_start: true,
265 token_starts: Vec::new(),
266 }
267 }
268
269 /// Byte offsets at which each produced token starts (parallel to the
270 /// caller's token stream).
271 pub fn token_starts(&self) -> &[usize] {
272 &self.token_starts
273 }
274
275 /// Advance to the next character.
276 fn advance(&mut self) {
277 self.pos += 1;
278 self.ch = if self.pos < self.input.len() {
279 self.input[self.pos]
280 } else {
281 0
282 };
283 }
284
285 /// Peek at the next character without consuming it.
286 fn peek(&self) -> u8 {
287 if self.pos + 1 < self.input.len() {
288 self.input[self.pos + 1]
289 } else {
290 0
291 }
292 }
293
294 /// Skip whitespace.
295 fn skip_ws(&mut self) {
296 while self.ch != 0
297 && (self.ch == b' ' || self.ch == b'\t' || self.ch == b'\n' || self.ch == b'\r')
298 {
299 self.advance();
300 }
301 }
302
303 /// Read a name token (NCName).
304 fn read_name(&mut self) -> String {
305 let start = self.pos;
306 while self.ch != 0
307 && (self.ch.is_ascii_alphanumeric()
308 || self.ch == b'_'
309 || self.ch == b'-'
310 || self.ch == b'.')
311 {
312 self.advance();
313 }
314 String::from_utf8_lossy(&self.input[start..self.pos]).to_string()
315 }
316
317 /// Try to match an axis name or keyword.
318 fn try_keyword_or_axis(&self, name: &str) -> Option<Token> {
319 match name {
320 "or" => Some(Token::Or),
321 "and" => Some(Token::And),
322 "mod" => Some(Token::Mod),
323 "div" => Some(Token::Div),
324 "ancestor" => Some(Token::Ancestor),
325 "ancestor-or-self" => Some(Token::AncestorOrSelf),
326 "attribute" => Some(Token::Attribute),
327 "child" => Some(Token::Child),
328 "descendant" => Some(Token::Descendant),
329 "descendant-or-self" => Some(Token::DescendantOrSelf),
330 "following" => Some(Token::Following),
331 "following-sibling" => Some(Token::FollowingSibling),
332 "namespace" => Some(Token::Namespace),
333 "parent" => Some(Token::Parent),
334 "preceding" => Some(Token::Preceding),
335 "preceding-sibling" => Some(Token::PrecedingSibling),
336 "self" => Some(Token::Self_),
337 _ => None,
338 }
339 }
340
341 /// Read a numeric literal — a faithful port of upstream xpath.c
342 /// `xmlXPathCompNumber` (R-000166). The oracle accumulates digits
343 /// directly (`ret = ret * 10 + d`), caps the fraction at MAX_FRAC=20
344 /// digits after any leading zeros, and applies the exponent with
345 /// `pow(10.0, exp)` — which underflows to 0 for exponents below the
346 /// smallest subnormal (e.g. `5e-324`). Rust's correctly-rounded
347 /// `strtod`-style parse differs in those edge cases, so the accumulation
348 /// is reproduced exactly.
349 fn read_number(&mut self) -> f64 {
350 let input = &self.input;
351 let len = input.len();
352 let mut cur = self.pos;
353
354 // Integer part.
355 let mut ret = 0.0f64;
356 while cur < len && input[cur].is_ascii_digit() {
357 ret = ret * 10.0 + (input[cur] - b'0') as f64;
358 cur += 1;
359 }
360
361 // Fractional part (upstream consumes a trailing '.' even without
362 // digits, so `5.` is a single number literal).
363 let mut frac: i32 = 0;
364 if cur < len && input[cur] == b'.' {
365 cur += 1;
366 while cur < len && input[cur] == b'0' {
367 frac += 1;
368 cur += 1;
369 }
370 let max = frac + 20; // MAX_FRAC
371 let mut fraction = 0.0f64;
372 while cur < len && input[cur].is_ascii_digit() && frac < max {
373 let v = (input[cur] - b'0') as f64;
374 fraction = fraction * 10.0 + v;
375 frac += 1;
376 cur += 1;
377 }
378 fraction /= 10f64.powf(frac as f64);
379 ret += fraction;
380 while cur < len && input[cur].is_ascii_digit() {
381 cur += 1;
382 }
383 }
384
385 // Exponent part (upstream xmlXPathCompNumber consumes 'e'/'E'
386 // unconditionally, then an optional sign, then digits — greedily
387 // even when malformed).
388 let mut exponent: i32 = 0;
389 let mut is_exponent_negative = false;
390 if cur < len && (input[cur] == b'e' || input[cur] == b'E') {
391 cur += 1;
392 if cur < len && input[cur] == b'-' {
393 is_exponent_negative = true;
394 cur += 1;
395 } else if cur < len && input[cur] == b'+' {
396 cur += 1;
397 }
398 while cur < len && input[cur].is_ascii_digit() {
399 if exponent < 1000000 {
400 exponent = exponent * 10 + (input[cur] - b'0') as i32;
401 }
402 cur += 1;
403 }
404 }
405 if is_exponent_negative {
406 exponent = -exponent;
407 }
408 ret *= 10f64.powf(exponent as f64);
409
410 self.pos = cur;
411 self.ch = if cur < len { input[cur] } else { 0 };
412 ret
413 }
414
415 /// Read a string literal.
416 fn read_string(&mut self, quote: u8) -> String {
417 self.advance(); // consume opening quote
418 let start = self.pos;
419 while self.ch != 0 && self.ch != quote {
420 self.advance();
421 }
422 let s = String::from_utf8_lossy(&self.input[start..self.pos]).to_string();
423 if self.ch == quote {
424 self.advance(); // consume closing quote
425 }
426 s
427 }
428
429 /// Get the next token.
430 pub fn next_token(&mut self) -> Token {
431 self.skip_ws();
432 // Record the token's byte offset (upstream `ctxt->cur - ctxt->base`
433 // for the XPath error caret).
434 self.token_starts.push(self.pos);
435
436 if self.ch == 0 {
437 return Token::Eof;
438 }
439
440 // Save at_start for unary minus detection
441 let _was_at_start = self.at_start;
442 self.at_start = false;
443
444 // ── Single-char tokens ────────────────────────────────────────────
445 match self.ch {
446 b'(' => {
447 self.advance();
448 return Token::LParen;
449 }
450 b')' => {
451 self.advance();
452 return Token::RParen;
453 }
454 b'[' => {
455 self.advance();
456 return Token::LBracket;
457 }
458 b']' => {
459 self.advance();
460 return Token::RBracket;
461 }
462 b'{' => {
463 self.advance();
464 return Token::LBrace;
465 }
466 b'}' => {
467 self.advance();
468 return Token::RBrace;
469 }
470 b',' => {
471 self.advance();
472 return Token::Comma;
473 }
474 b'$' => {
475 self.advance();
476 return Token::Dollar;
477 }
478 b'|' => {
479 self.advance();
480 return Token::Pipe;
481 }
482 b'+' => {
483 self.advance();
484 return Token::Plus;
485 }
486 b'@' => {
487 self.advance();
488 return Token::At;
489 }
490 b'.' => {
491 if self.peek() == b'.' {
492 self.advance();
493 self.advance();
494 return Token::DotDot;
495 }
496 // Check if it's a number starting with '.'
497 if self.peek().is_ascii_digit() {
498 return Token::NumberLiteral(self.read_number());
499 }
500 self.advance();
501 return Token::Dot;
502 }
503 b'-' => {
504 self.advance();
505 // If at start or after operator, this is unary minus
506 // We handle this at the parser level, just return Minus
507 return Token::Minus;
508 }
509 b'=' => {
510 self.advance();
511 return Token::Eq;
512 }
513 b'!' => {
514 if self.peek() == b'=' {
515 self.advance();
516 self.advance();
517 return Token::Ne;
518 }
519 // Invalid character, skip
520 self.advance();
521 return self.next_token();
522 }
523 b'<' => {
524 self.advance();
525 if self.ch == b'=' {
526 self.advance();
527 return Token::Le;
528 }
529 return Token::Lt;
530 }
531 b'>' => {
532 self.advance();
533 if self.ch == b'=' {
534 self.advance();
535 return Token::Ge;
536 }
537 return Token::Gt;
538 }
539 b'/' => {
540 self.advance();
541 if self.ch == b'/' {
542 self.advance();
543 return Token::DoubleSlash;
544 }
545 return Token::Slash;
546 }
547 b'*' => {
548 self.advance();
549 return Token::Star; // lexer returns Star; parser disambiguates
550 }
551 b':' => {
552 if self.peek() == b':' {
553 self.advance();
554 self.advance();
555 return Token::DoubleColon;
556 }
557 // Single colon is part of a QName, handled below
558 // Actually, if we see a colon, it should be part of a name
559 // This case handles axis::name or prefix:name
560 // Since we read the full name first, this shouldn't normally happen alone
561 self.advance();
562 return self.next_token();
563 }
564 b'\'' | b'"' => {
565 let quote = self.ch;
566 let s = self.read_string(quote);
567 return Token::StringLiteral(s);
568 }
569 _ => {}
570 }
571
572 // ── Number ───────────────────────────────────────────────────────
573 if self.ch.is_ascii_digit() {
574 return Token::NumberLiteral(self.read_number());
575 }
576
577 // ── Name ─────────────────────────────────────────────────────────
578 if self.ch.is_ascii_alphabetic() || self.ch == b'_' {
579 let name = self.read_name();
580
581 // Check for QName (prefix:local)
582 if self.ch == b':' && self.peek() != b':' {
583 self.advance(); // consume ':'
584 if self.ch.is_ascii_alphabetic() || self.ch == b'_' || self.ch == b'*' {
585 if self.ch == b'*' {
586 self.advance();
587 let full = format!("{}:*", name);
588 return Token::Name(full);
589 }
590 let local = self.read_name();
591 return Token::Name(format!("{}:{}", name, local));
592 }
593 // If the colon is not followed by a valid name character,
594 // it might be an axis separator that got split. Push back?
595 // Actually in well-formed XPath, `name:` is followed by `:`
596 // for axis:: or by a local name for QName.
597 // We already checked peek != ':', so this is a QName prefix.
598 // If the local part is missing, treat the whole thing as a name.
599 return Token::Name(name);
600 }
601
602 // Check for axis separator: name::
603 // We DON'T consume the :: here — we return just the axis keyword token.
604 // The :: will be tokenized as DoubleColon on the next call to next_token().
605 if self.ch == b':' && self.peek() == b':' {
606 if let Some(axis) = self.try_keyword_or_axis(&name) {
607 return axis;
608 }
609 // Not an axis keyword — could be a QName prefix followed by ::?
610 // Treat it as a regular name and let the :: be consumed separately.
611 return Token::Name(name);
612 }
613
614 // Check for keyword or axis
615 if let Some(keyword) = self.try_keyword_or_axis(&name) {
616 return keyword;
617 }
618
619 return Token::Name(name);
620 }
621
622 // Unknown character, skip
623 self.advance();
624 self.next_token()
625 }
626}
627
628// ═══════════════════════════════════════════════════════════════════════════════
629// Tests
630// ═══════════════════════════════════════════════════════════════════════════════
631
632#[cfg(test)]
633mod tests {
634 use super::*;
635
636 fn tokenize(s: &str) -> Vec<Token> {
637 let mut lexer = Lexer::new(s);
638 let mut tokens = Vec::new();
639 loop {
640 let tok = lexer.next_token();
641 let is_eof = matches!(tok, Token::Eof);
642 tokens.push(tok);
643 if is_eof {
644 break;
645 }
646 }
647 tokens
648 }
649
650 #[test]
651 fn test_empty() {
652 let tokens = tokenize("");
653 assert_eq!(tokens.len(), 1);
654 assert_eq!(tokens[0], Token::Eof);
655 }
656
657 #[test]
658 fn test_simple_path() {
659 let tokens = tokenize("child::para");
660 assert_eq!(
661 tokens,
662 vec![
663 Token::Child,
664 Token::DoubleColon,
665 Token::Name("para".into()),
666 Token::Eof,
667 ]
668 );
669 }
670
671 #[test]
672 fn test_absolute_path() {
673 let tokens = tokenize("/child::para");
674 assert_eq!(
675 tokens,
676 vec![
677 Token::Slash,
678 Token::Child,
679 Token::DoubleColon,
680 Token::Name("para".into()),
681 Token::Eof,
682 ]
683 );
684 }
685
686 #[test]
687 fn test_short_form() {
688 let tokens = tokenize("para");
689 assert_eq!(tokens, vec![Token::Name("para".into()), Token::Eof]);
690 }
691
692 #[test]
693 fn test_attribute() {
694 let tokens = tokenize("@attr");
695 assert_eq!(
696 tokens,
697 vec![Token::At, Token::Name("attr".into()), Token::Eof]
698 );
699 }
700
701 #[test]
702 fn test_predicate() {
703 let tokens = tokenize("para[1]");
704 assert_eq!(
705 tokens,
706 vec![
707 Token::Name("para".into()),
708 Token::LBracket,
709 Token::NumberLiteral(1.0),
710 Token::RBracket,
711 Token::Eof,
712 ]
713 );
714 }
715
716 #[test]
717 fn test_function_call() {
718 let tokens = tokenize("position()");
719 assert_eq!(
720 tokens,
721 vec![
722 Token::Name("position".into()),
723 Token::LParen,
724 Token::RParen,
725 Token::Eof,
726 ]
727 );
728 }
729
730 #[test]
731 fn test_string_literal() {
732 let tokens = tokenize("'hello'");
733 assert_eq!(
734 tokens,
735 vec![Token::StringLiteral("hello".into()), Token::Eof]
736 );
737 }
738
739 #[test]
740 fn test_number() {
741 let tokens = tokenize("42");
742 assert_eq!(tokens, vec![Token::NumberLiteral(42.0), Token::Eof]);
743 }
744 #[allow(clippy::approx_constant)]
745 #[test]
746 fn test_decimal() {
747 let tokens = tokenize("3.14");
748 assert_eq!(tokens, vec![Token::NumberLiteral(3.14), Token::Eof]);
749 }
750
751 #[test]
752 fn test_operators() {
753 let tokens = tokenize("a = b and c != d or e < f");
754 assert!(tokens.contains(&Token::Eq));
755 assert!(tokens.contains(&Token::And));
756 assert!(tokens.contains(&Token::Ne));
757 assert!(tokens.contains(&Token::Or));
758 assert!(tokens.contains(&Token::Lt));
759 }
760
761 #[test]
762 fn test_union() {
763 let tokens = tokenize("a | b");
764 assert_eq!(
765 tokens,
766 vec![
767 Token::Name("a".into()),
768 Token::Pipe,
769 Token::Name("b".into()),
770 Token::Eof,
771 ]
772 );
773 }
774
775 #[test]
776 fn test_double_slash() {
777 let tokens = tokenize("//para");
778 assert_eq!(
779 tokens,
780 vec![Token::DoubleSlash, Token::Name("para".into()), Token::Eof]
781 );
782 }
783
784 #[test]
785 fn test_qname() {
786 let tokens = tokenize("xslt:template");
787 assert_eq!(
788 tokens,
789 vec![Token::Name("xslt:template".into()), Token::Eof]
790 );
791 }
792
793 #[test]
794 fn test_wildcard() {
795 let tokens = tokenize("*");
796 assert_eq!(tokens, vec![Token::Star, Token::Eof]);
797 }
798
799 #[test]
800 fn test_ns_wildcard() {
801 let tokens = tokenize("ns:*");
802 assert_eq!(tokens, vec![Token::Name("ns:*".into()), Token::Eof]);
803 }
804
805 #[test]
806 fn test_dot_dot() {
807 let tokens = tokenize("..");
808 assert_eq!(tokens, vec![Token::DotDot, Token::Eof]);
809 }
810
811 #[test]
812 fn test_axis_keyword() {
813 let tokens = tokenize("ancestor-or-self::node()");
814 assert_eq!(
815 tokens,
816 vec![
817 Token::AncestorOrSelf,
818 Token::DoubleColon,
819 Token::Name("node".into()),
820 Token::LParen,
821 Token::RParen,
822 Token::Eof,
823 ]
824 );
825 }
826
827 #[test]
828 fn test_complex_expression() {
829 let tokens = tokenize("/html/body//div[@class='main']/p[1]");
830 // Collect name-like tokens (including keyword tokens that can be element names)
831 let names: Vec<String> = tokens
832 .iter()
833 .filter_map(|t| match t {
834 Token::Name(n) => Some(n.clone()),
835 Token::Div => Some("div".to_string()),
836 Token::Mod => Some("mod".to_string()),
837 Token::And => Some("and".to_string()),
838 Token::Or => Some("or".to_string()),
839 _ => None,
840 })
841 .collect();
842 assert_eq!(names, vec!["html", "body", "div", "class", "p"]);
843 }
844
845 #[test]
846 fn test_variable() {
847 let tokens = tokenize("$var");
848 assert_eq!(
849 tokens,
850 vec![Token::Dollar, Token::Name("var".into()), Token::Eof]
851 );
852 }
853}