perl_token/
lib.rs

1//! Perl token definitions shared across the parser ecosystem.
2//!
3//! This crate defines [`Token`] and [`TokenKind`], the fundamental types that
4//! flow from the lexer (`perl-lexer`) into the parser (`perl-parser-core`).
5//! Downstream crates re-export these types so consumers rarely need to depend
6//! on `perl-token` directly.
7//!
8//! # Examples
9//!
10//! Create and inspect tokens:
11//!
12//! ```rust
13//! use perl_token::{Token, TokenKind};
14//!
15//! // Create a keyword token for `my`
16//! let token = Token::new(TokenKind::My, "my", 0, 2);
17//! assert_eq!(token.kind, TokenKind::My);
18//! assert_eq!(&*token.text, "my");
19//! assert_eq!(token.start, 0);
20//! assert_eq!(token.end, 2);
21//!
22//! // Create a numeric literal token
23//! let num = Token::new(TokenKind::Number, "42", 7, 9);
24//! assert_eq!(num.kind, TokenKind::Number);
25//! assert_eq!(&*num.text, "42");
26//! ```
27//!
28//! Use [`TokenKind::display_name`] for user-facing error messages:
29//!
30//! ```rust
31//! use perl_token::TokenKind;
32//!
33//! assert_eq!(TokenKind::LeftBrace.display_name(), "'{'");
34//! assert_eq!(TokenKind::Identifier.display_name(), "identifier");
35//! assert_eq!(TokenKind::Eof.display_name(), "end of input");
36//! ```
37
38#![warn(missing_docs)]
39
40mod kind;
41mod token;
42
43pub use kind::{
44    DELIMITER_SPELLINGS, KEYWORD_SPELLINGS, OPERATOR_SPELLINGS, SIGIL_SPELLINGS, TokenCategory,
45    TokenKind, TokenKindMetadata,
46};
47pub use token::{Token, TokenRef, TokenSpan, TokenSpanError};
48
49#[cfg(test)]
50mod tests {
51    use super::*;
52
53    // --- TokenSpan ---
54
55    #[test]
56    fn token_span_new_and_accessors() {
57        let span = TokenSpan::new(5, 10);
58        assert_eq!(span.start, 5);
59        assert_eq!(span.end, 10);
60        assert_eq!(span.len(), 5);
61        assert!(!span.is_empty());
62        assert_eq!(span.range(), 5..10);
63    }
64
65    #[test]
66    fn token_span_is_empty_when_zero_length() {
67        let span = TokenSpan::new(3, 3);
68        assert!(span.is_empty());
69        assert_eq!(span.len(), 0);
70    }
71
72    #[test]
73    fn token_span_try_new_ok() -> Result<(), TokenSpanError> {
74        let span = TokenSpan::try_new(0, 5)?;
75        assert_eq!(span.start, 0);
76        assert_eq!(span.end, 5);
77        Ok(())
78    }
79
80    #[test]
81    fn token_span_try_new_end_before_start_errors() {
82        assert_eq!(
83            TokenSpan::try_new(10, 5),
84            Err(TokenSpanError::EndBeforeStart { start: 10, end: 5 })
85        );
86    }
87
88    #[test]
89    fn token_span_error_display_end_before_start() {
90        let err = TokenSpanError::EndBeforeStart { start: 10, end: 5 };
91        let msg = err.to_string();
92        assert!(msg.contains("10"));
93        assert!(msg.contains("5"));
94    }
95
96    #[test]
97    fn token_span_error_display_empty_span_not_allowed() {
98        let err = TokenSpanError::EmptySpanNotAllowed { kind: TokenKind::Identifier, at: 7 };
99        let msg = err.to_string();
100        assert!(msg.contains("Identifier"));
101        assert!(msg.contains("7"));
102    }
103
104    // --- Token ---
105
106    #[test]
107    fn token_new_stores_fields() {
108        let tok = Token::new(TokenKind::My, "my", 0, 2);
109        assert_eq!(tok.kind, TokenKind::My);
110        assert_eq!(&*tok.text, "my");
111        assert_eq!(tok.start, 0);
112        assert_eq!(tok.end, 2);
113    }
114
115    #[test]
116    fn token_len_saturates_for_inverted_span() {
117        let tok = Token::new(TokenKind::Identifier, "x", 9, 4);
118        assert_eq!(tok.len(), 0);
119        assert!(tok.is_empty());
120    }
121
122    #[test]
123    fn token_len_and_is_empty() {
124        let tok = Token::new(TokenKind::Identifier, "foo", 10, 13);
125        assert_eq!(tok.len(), 3);
126        assert!(!tok.is_empty());
127
128        let eof = Token::eof_at(8);
129        assert_eq!(eof.len(), 0);
130        assert!(eof.is_empty());
131    }
132
133    #[test]
134    fn token_span_and_range() {
135        let tok = Token::new(TokenKind::Number, "42", 5, 7);
136        assert_eq!(tok.span(), TokenSpan::new(5, 7));
137        assert_eq!(tok.range(), 5..7);
138    }
139
140    #[test]
141    fn token_try_new_allows_ordered_spans() -> Result<(), TokenSpanError> {
142        let tok = Token::try_new(TokenKind::Identifier, "name", 4, 8)?;
143        assert_eq!(tok.kind, TokenKind::Identifier);
144        assert_eq!(&*tok.text, "name");
145        assert_eq!(tok.span(), TokenSpan::new(4, 8));
146        Ok(())
147    }
148
149    #[test]
150    fn token_try_new_rejects_end_before_start() {
151        assert_eq!(
152            Token::try_new(TokenKind::Identifier, "x", 10, 5),
153            Err(TokenSpanError::EndBeforeStart { start: 10, end: 5 })
154        );
155    }
156
157    #[test]
158    fn token_new_checked_rejects_empty_non_eof() {
159        assert_eq!(
160            Token::new_checked(TokenKind::Identifier, "", 5, 5),
161            Err(TokenSpanError::EmptySpanNotAllowed { kind: TokenKind::Identifier, at: 5 })
162        );
163    }
164
165    #[test]
166    fn token_new_checked_allows_empty_eof() -> Result<(), TokenSpanError> {
167        let tok = Token::new_checked(TokenKind::Eof, "", 5, 5)?;
168        assert_eq!(tok.kind, TokenKind::Eof);
169        assert_eq!(tok.start, 5);
170        Ok(())
171    }
172
173    #[test]
174    fn token_new_checked_allows_empty_unknown() -> Result<(), TokenSpanError> {
175        let tok = Token::new_checked(TokenKind::Unknown, "", 6, 6)?;
176        assert_eq!(tok.kind, TokenKind::Unknown);
177        assert_eq!(tok.start, 6);
178        assert!(tok.is_empty());
179        Ok(())
180    }
181
182    #[test]
183    fn token_eof_at() {
184        let eof = Token::eof_at(42);
185        assert_eq!(eof.kind, TokenKind::Eof);
186        assert_eq!(eof.start, 42);
187        assert_eq!(eof.end, 42);
188        assert!(eof.is_empty());
189    }
190
191    #[test]
192    fn token_unknown_at_normalises_inverted_span() {
193        let tok = Token::unknown_at("?", 5, 3); // end < start
194        assert_eq!(tok.kind, TokenKind::Unknown);
195        assert_eq!(tok.start, 5);
196        assert_eq!(tok.end, 5); // bounded to start
197    }
198
199    #[test]
200    fn token_with_kind() {
201        let tok = Token::new(TokenKind::Identifier, "sub", 0, 3);
202        let retyped = tok.with_kind(TokenKind::Sub);
203        assert_eq!(retyped.kind, TokenKind::Sub);
204        assert_eq!(&*retyped.text, "sub");
205        assert_eq!(retyped.start, 0);
206        assert_eq!(retyped.end, 3);
207    }
208
209    #[test]
210    fn token_with_span_ok() -> Result<(), TokenSpanError> {
211        let tok = Token::new(TokenKind::String, "hello", 0, 5);
212        let moved = tok.with_span(10, 15)?;
213        assert_eq!(moved.start, 10);
214        assert_eq!(moved.end, 15);
215        Ok(())
216    }
217
218    #[test]
219    fn token_with_span_rejects_empty_non_eof() {
220        let tok = Token::new(TokenKind::String, "hello", 0, 5);
221        assert_eq!(
222            tok.with_span(10, 10),
223            Err(TokenSpanError::EmptySpanNotAllowed { kind: TokenKind::String, at: 10 })
224        );
225    }
226
227    #[test]
228    fn token_display_name_delegates_to_kind() {
229        let tok = Token::new(TokenKind::LeftBrace, "{", 0, 1);
230        assert_eq!(tok.display_name(), "'{'");
231    }
232
233    #[test]
234    fn token_as_ref_token_round_trip() {
235        let tok = Token::new(TokenKind::Sub, "sub", 0, 3);
236        let tok_ref = tok.as_ref_token();
237        assert_eq!(tok_ref.kind, TokenKind::Sub);
238        assert_eq!(tok_ref.text, "sub");
239        assert_eq!(tok_ref.start, 0);
240        assert_eq!(tok_ref.end, 3);
241
242        let owned: Token = tok_ref.into();
243        assert_eq!(owned.kind, TokenKind::Sub);
244        assert_eq!(&*owned.text, "sub");
245    }
246
247    // --- TokenRef ---
248
249    #[test]
250    fn token_ref_len_saturates_for_inverted_span() {
251        let r = TokenRef::new(TokenKind::Identifier, "x", 9, 4);
252        assert_eq!(r.len(), 0);
253        assert!(r.is_empty());
254    }
255
256    #[test]
257    fn token_ref_accessors() {
258        let r = TokenRef::new(TokenKind::Number, "99", 4, 6);
259        assert_eq!(r.len(), 2);
260        assert!(!r.is_empty());
261        assert_eq!(r.span(), (4, 6));
262        assert_eq!(r.display_name(), "number");
263    }
264
265    #[test]
266    fn token_ref_try_new_allows_ordered_spans() -> Result<(), TokenSpanError> {
267        let r = TokenRef::try_new(TokenKind::Number, "99", 4, 6)?;
268        assert_eq!(r.kind, TokenKind::Number);
269        assert_eq!(r.text, "99");
270        assert_eq!(r.span(), (4, 6));
271        Ok(())
272    }
273
274    #[test]
275    fn token_ref_to_owned_token() {
276        let r = TokenRef::new(TokenKind::Identifier, "foo", 1, 4);
277        let owned = r.to_owned_token();
278        assert_eq!(owned.kind, TokenKind::Identifier);
279        assert_eq!(&*owned.text, "foo");
280    }
281
282    // --- TokenKind::from_keyword ---
283
284    #[test]
285    fn from_keyword_recognises_perl_keywords() {
286        assert_eq!(TokenKind::from_keyword("my"), Some(TokenKind::My));
287        assert_eq!(TokenKind::from_keyword("sub"), Some(TokenKind::Sub));
288        assert_eq!(TokenKind::from_keyword("if"), Some(TokenKind::If));
289        assert_eq!(TokenKind::from_keyword("elsif"), Some(TokenKind::Elsif));
290        assert_eq!(TokenKind::from_keyword("else"), Some(TokenKind::Else));
291        assert_eq!(TokenKind::from_keyword("while"), Some(TokenKind::While));
292        assert_eq!(TokenKind::from_keyword("for"), Some(TokenKind::For));
293        assert_eq!(TokenKind::from_keyword("foreach"), Some(TokenKind::Foreach));
294        assert_eq!(TokenKind::from_keyword("return"), Some(TokenKind::Return));
295        assert_eq!(TokenKind::from_keyword("package"), Some(TokenKind::Package));
296        assert_eq!(TokenKind::from_keyword("use"), Some(TokenKind::Use));
297        assert_eq!(TokenKind::from_keyword("BEGIN"), Some(TokenKind::Begin));
298        assert_eq!(TokenKind::from_keyword("END"), Some(TokenKind::End));
299        assert_eq!(TokenKind::from_keyword("eval"), Some(TokenKind::Eval));
300        assert_eq!(TokenKind::from_keyword("class"), Some(TokenKind::Class));
301        assert_eq!(TokenKind::from_keyword("defer"), Some(TokenKind::Defer));
302        assert_eq!(TokenKind::from_keyword("and"), Some(TokenKind::WordAnd));
303        assert_eq!(TokenKind::from_keyword("or"), Some(TokenKind::WordOr));
304        assert_eq!(TokenKind::from_keyword("not"), Some(TokenKind::WordNot));
305        assert_eq!(TokenKind::from_keyword("xor"), Some(TokenKind::WordXor));
306        assert_eq!(TokenKind::from_keyword("cmp"), Some(TokenKind::StringCompare));
307    }
308
309    #[test]
310    fn from_keyword_unknown_returns_none() {
311        assert_eq!(TokenKind::from_keyword("MY"), None);
312        assert_eq!(TokenKind::from_keyword("Sub"), None);
313        assert_eq!(TokenKind::from_keyword("unknown"), None);
314        assert_eq!(TokenKind::from_keyword(""), None);
315    }
316
317    // --- TokenKind::from_operator ---
318
319    #[test]
320    fn from_operator_recognises_operators() {
321        assert_eq!(TokenKind::from_operator("="), Some(TokenKind::Assign));
322        assert_eq!(TokenKind::from_operator("+"), Some(TokenKind::Plus));
323        assert_eq!(TokenKind::from_operator("**"), Some(TokenKind::Power));
324        assert_eq!(TokenKind::from_operator("->"), Some(TokenKind::Arrow));
325        assert_eq!(TokenKind::from_operator("=>"), Some(TokenKind::FatArrow));
326        assert_eq!(TokenKind::from_operator("<=>"), Some(TokenKind::Spaceship));
327        assert_eq!(TokenKind::from_operator("//="), Some(TokenKind::DefinedOrAssign));
328        assert_eq!(TokenKind::from_operator("..."), Some(TokenKind::Ellipsis));
329        assert_eq!(TokenKind::from_operator("~~"), Some(TokenKind::SmartMatch));
330    }
331
332    #[test]
333    fn from_operator_unknown_returns_none() {
334        assert_eq!(TokenKind::from_operator(""), None);
335        assert_eq!(TokenKind::from_operator("xyz"), None);
336    }
337
338    // --- TokenKind::from_delimiter ---
339
340    #[test]
341    fn from_delimiter_recognises_all() {
342        assert_eq!(TokenKind::from_delimiter("("), Some(TokenKind::LeftParen));
343        assert_eq!(TokenKind::from_delimiter(")"), Some(TokenKind::RightParen));
344        assert_eq!(TokenKind::from_delimiter("{"), Some(TokenKind::LeftBrace));
345        assert_eq!(TokenKind::from_delimiter("}"), Some(TokenKind::RightBrace));
346        assert_eq!(TokenKind::from_delimiter("["), Some(TokenKind::LeftBracket));
347        assert_eq!(TokenKind::from_delimiter("]"), Some(TokenKind::RightBracket));
348        assert_eq!(TokenKind::from_delimiter(";"), Some(TokenKind::Semicolon));
349        assert_eq!(TokenKind::from_delimiter(","), Some(TokenKind::Comma));
350        assert_eq!(TokenKind::from_delimiter("x"), None);
351    }
352
353    // --- TokenKind::from_sigil ---
354
355    #[test]
356    fn from_sigil_recognises_all() {
357        assert_eq!(TokenKind::from_sigil("$"), Some(TokenKind::ScalarSigil));
358        assert_eq!(TokenKind::from_sigil("@"), Some(TokenKind::ArraySigil));
359        assert_eq!(TokenKind::from_sigil("%"), Some(TokenKind::HashSigil));
360        assert_eq!(TokenKind::from_sigil("&"), Some(TokenKind::SubSigil));
361        assert_eq!(TokenKind::from_sigil("*"), Some(TokenKind::GlobSigil));
362        assert_eq!(TokenKind::from_sigil("!"), None);
363    }
364
365    // --- TokenKind::category ---
366
367    #[test]
368    fn category_keyword_variants() {
369        assert_eq!(TokenKind::My.category(), TokenCategory::Keyword);
370        assert_eq!(TokenKind::Sub.category(), TokenCategory::Keyword);
371        assert_eq!(TokenKind::Defer.category(), TokenCategory::Keyword);
372    }
373
374    #[test]
375    fn category_operator_variants() {
376        assert_eq!(TokenKind::Plus.category(), TokenCategory::Operator);
377        assert_eq!(TokenKind::Spaceship.category(), TokenCategory::Operator);
378        assert_eq!(TokenKind::WordAnd.category(), TokenCategory::Operator);
379    }
380
381    #[test]
382    fn category_delimiter_variants() {
383        assert_eq!(TokenKind::LeftParen.category(), TokenCategory::Delimiter);
384        assert_eq!(TokenKind::Comma.category(), TokenCategory::Delimiter);
385    }
386
387    #[test]
388    fn category_literal_variants() {
389        assert_eq!(TokenKind::Number.category(), TokenCategory::Literal);
390        assert_eq!(TokenKind::HeredocStart.category(), TokenCategory::Literal);
391        assert_eq!(TokenKind::DataMarker.category(), TokenCategory::Literal);
392    }
393
394    #[test]
395    fn category_identifier_variants() {
396        assert_eq!(TokenKind::Identifier.category(), TokenCategory::Identifier);
397        assert_eq!(TokenKind::ScalarSigil.category(), TokenCategory::Identifier);
398        assert_eq!(TokenKind::GlobSigil.category(), TokenCategory::Identifier);
399    }
400
401    #[test]
402    fn category_special_variants() {
403        assert_eq!(TokenKind::Eof.category(), TokenCategory::Special);
404        assert_eq!(TokenKind::Unknown.category(), TokenCategory::Special);
405    }
406
407    // --- TokenKind::display_name ---
408
409    #[test]
410    fn display_name_selected_variants() {
411        assert_eq!(TokenKind::LeftBrace.display_name(), "'{'");
412        assert_eq!(TokenKind::RightBrace.display_name(), "'}'");
413        assert_eq!(TokenKind::Identifier.display_name(), "identifier");
414        assert_eq!(TokenKind::Eof.display_name(), "end of input");
415        assert_eq!(TokenKind::Number.display_name(), "number");
416        assert_eq!(TokenKind::Sub.display_name(), "'sub'");
417        assert_eq!(TokenKind::Semicolon.display_name(), "';'");
418        assert_eq!(TokenKind::HeredocStart.display_name(), "heredoc (<<)");
419        assert_eq!(TokenKind::DataMarker.display_name(), "data marker (__DATA__ or __END__)");
420    }
421
422    // --- TokenKind::all / metadata_count ---
423
424    #[test]
425    fn all_returns_132_variants() {
426        assert_eq!(TokenKind::all().len(), 132);
427        assert_eq!(TokenKind::metadata_count(), 132);
428    }
429
430    #[test]
431    fn metadata_round_trips_through_kind() {
432        let m = TokenKind::Sub.metadata();
433        assert_eq!(m.category, TokenCategory::Keyword);
434        assert_eq!(m.display_name, "'sub'");
435    }
436
437    // --- TokenKind role predicates ---
438
439    #[test]
440    fn is_assignment_operator_returns_true_for_assign_variants() {
441        assert!(TokenKind::Assign.is_assignment_operator());
442        assert!(TokenKind::PlusAssign.is_assignment_operator());
443        assert!(TokenKind::MinusAssign.is_assignment_operator());
444        assert!(TokenKind::StarAssign.is_assignment_operator());
445        assert!(TokenKind::SlashAssign.is_assignment_operator());
446        assert!(TokenKind::PercentAssign.is_assignment_operator());
447        assert!(TokenKind::DotAssign.is_assignment_operator());
448        assert!(TokenKind::AndAssign.is_assignment_operator());
449        assert!(TokenKind::OrAssign.is_assignment_operator());
450        assert!(TokenKind::XorAssign.is_assignment_operator());
451        assert!(TokenKind::PowerAssign.is_assignment_operator());
452        assert!(TokenKind::LeftShiftAssign.is_assignment_operator());
453        assert!(TokenKind::RightShiftAssign.is_assignment_operator());
454        assert!(TokenKind::LogicalAndAssign.is_assignment_operator());
455        assert!(TokenKind::LogicalOrAssign.is_assignment_operator());
456        assert!(TokenKind::DefinedOrAssign.is_assignment_operator());
457    }
458
459    #[test]
460    fn is_assignment_operator_returns_false_for_non_assign() {
461        assert!(!TokenKind::Plus.is_assignment_operator());
462        assert!(!TokenKind::Equal.is_assignment_operator());
463        assert!(!TokenKind::Identifier.is_assignment_operator());
464    }
465
466    #[test]
467    fn is_logical_operator_returns_true_for_logical_variants() {
468        assert!(TokenKind::And.is_logical_operator());
469        assert!(TokenKind::Or.is_logical_operator());
470        assert!(TokenKind::Not.is_logical_operator());
471        assert!(TokenKind::DefinedOr.is_logical_operator());
472        assert!(TokenKind::WordAnd.is_logical_operator());
473        assert!(TokenKind::WordOr.is_logical_operator());
474        assert!(TokenKind::WordNot.is_logical_operator());
475        assert!(TokenKind::WordXor.is_logical_operator());
476    }
477
478    #[test]
479    fn is_logical_operator_returns_false_for_non_logical() {
480        assert!(!TokenKind::Plus.is_logical_operator());
481        assert!(!TokenKind::Assign.is_logical_operator());
482        assert!(!TokenKind::Identifier.is_logical_operator());
483    }
484
485    #[test]
486    fn is_open_delimiter_returns_true_for_open_delimiters() {
487        assert!(TokenKind::LeftParen.is_open_delimiter());
488        assert!(TokenKind::LeftBrace.is_open_delimiter());
489        assert!(TokenKind::LeftBracket.is_open_delimiter());
490    }
491
492    #[test]
493    fn is_open_delimiter_returns_false_for_non_open() {
494        assert!(!TokenKind::RightParen.is_open_delimiter());
495        assert!(!TokenKind::Semicolon.is_open_delimiter());
496        assert!(!TokenKind::Plus.is_open_delimiter());
497    }
498
499    #[test]
500    fn is_quote_like_returns_true_for_quote_variants() {
501        assert!(TokenKind::Regex.is_quote_like());
502        assert!(TokenKind::Substitution.is_quote_like());
503        assert!(TokenKind::Transliteration.is_quote_like());
504        assert!(TokenKind::QuoteSingle.is_quote_like());
505        assert!(TokenKind::QuoteDouble.is_quote_like());
506        assert!(TokenKind::QuoteWords.is_quote_like());
507        assert!(TokenKind::QuoteCommand.is_quote_like());
508        assert!(TokenKind::HeredocStart.is_quote_like());
509    }
510
511    #[test]
512    fn is_quote_like_returns_false_for_non_quote() {
513        assert!(!TokenKind::String.is_quote_like());
514        assert!(!TokenKind::Identifier.is_quote_like());
515        assert!(!TokenKind::LeftParen.is_quote_like());
516    }
517
518    #[test]
519    fn is_recovery_boundary_returns_true_for_boundaries() {
520        assert!(TokenKind::Semicolon.is_recovery_boundary());
521        assert!(TokenKind::RightParen.is_recovery_boundary());
522        assert!(TokenKind::RightBrace.is_recovery_boundary());
523        assert!(TokenKind::RightBracket.is_recovery_boundary());
524        assert!(TokenKind::Eof.is_recovery_boundary());
525    }
526
527    #[test]
528    fn is_recovery_boundary_returns_false_for_non_boundary() {
529        assert!(!TokenKind::Plus.is_recovery_boundary());
530        assert!(!TokenKind::Identifier.is_recovery_boundary());
531        assert!(!TokenKind::LeftParen.is_recovery_boundary());
532    }
533
534    // --- TokenRef::new_checked branches ---
535
536    #[test]
537    fn token_ref_new_checked_rejects_end_before_start() {
538        assert_eq!(
539            TokenRef::new_checked(TokenKind::Identifier, "x", 10, 3),
540            Err(TokenSpanError::EndBeforeStart { start: 10, end: 3 })
541        );
542    }
543
544    #[test]
545    fn token_ref_new_checked_allows_empty_eof() -> Result<(), Box<dyn std::error::Error>> {
546        let tok = TokenRef::new_checked(TokenKind::Eof, "", 7, 7)?;
547        assert_eq!(tok.kind, TokenKind::Eof);
548        assert_eq!(tok.start, 7);
549        assert!(tok.is_empty());
550        Ok(())
551    }
552
553    #[test]
554    fn token_ref_new_checked_allows_empty_unknown() -> Result<(), Box<dyn std::error::Error>> {
555        let tok = TokenRef::new_checked(TokenKind::Unknown, "", 3, 3)?;
556        assert_eq!(tok.kind, TokenKind::Unknown);
557        assert_eq!(tok.start, 3);
558        assert!(tok.is_empty());
559        Ok(())
560    }
561
562    #[test]
563    fn token_ref_new_checked_rejects_empty_non_eof() {
564        assert_eq!(
565            TokenRef::new_checked(TokenKind::Identifier, "", 5, 5),
566            Err(TokenSpanError::EmptySpanNotAllowed { kind: TokenKind::Identifier, at: 5 })
567        );
568    }
569}
perl_token/lib.rs

perl_token/
lib.rs
