perl_token/
lib.rs

1//! Perl token definitions shared across the parser ecosystem.
2//!
3//! This crate defines [`Token`] and [`TokenKind`], the fundamental types that
4//! flow from the lexer (`perl-lexer`) into the parser (`perl-parser-core`).
5//! Downstream crates re-export these types so consumers rarely need to depend
6//! on `perl-token` directly.
7//!
8//! # Examples
9//!
10//! Create and inspect tokens:
11//!
12//! ```rust
13//! use perl_token::{Token, TokenKind};
14//!
15//! // Create a keyword token for `my`
16//! let token = Token::new(TokenKind::My, "my", 0, 2);
17//! assert_eq!(token.kind, TokenKind::My);
18//! assert_eq!(&*token.text, "my");
19//! assert_eq!(token.start, 0);
20//! assert_eq!(token.end, 2);
21//!
22//! // Create a numeric literal token
23//! let num = Token::new(TokenKind::Number, "42", 7, 9);
24//! assert_eq!(num.kind, TokenKind::Number);
25//! assert_eq!(&*num.text, "42");
26//! ```
27//!
28//! Use [`TokenKind::display_name`] for user-facing error messages:
29//!
30//! ```rust
31//! use perl_token::TokenKind;
32//!
33//! assert_eq!(TokenKind::LeftBrace.display_name(), "'{'");
34//! assert_eq!(TokenKind::Identifier.display_name(), "identifier");
35//! assert_eq!(TokenKind::Eof.display_name(), "end of input");
36//! ```
37
38mod kind;
39mod token;
40
41pub use kind::{
42    DELIMITER_SPELLINGS, KEYWORD_SPELLINGS, OPERATOR_SPELLINGS, SIGIL_SPELLINGS, TokenCategory,
43    TokenKind, TokenKindMetadata,
44};
45pub use token::{Token, TokenRef, TokenSpan, TokenSpanError};
46
47#[cfg(test)]
48mod tests {
49    use super::*;
50
51    // --- TokenSpan ---
52
53    #[test]
54    fn token_span_new_and_accessors() {
55        let span = TokenSpan::new(5, 10);
56        assert_eq!(span.start, 5);
57        assert_eq!(span.end, 10);
58        assert_eq!(span.len(), 5);
59        assert!(!span.is_empty());
60        assert_eq!(span.range(), 5..10);
61    }
62
63    #[test]
64    fn token_span_is_empty_when_zero_length() {
65        let span = TokenSpan::new(3, 3);
66        assert!(span.is_empty());
67        assert_eq!(span.len(), 0);
68    }
69
70    #[test]
71    fn token_span_try_new_ok() -> Result<(), TokenSpanError> {
72        let span = TokenSpan::try_new(0, 5)?;
73        assert_eq!(span.start, 0);
74        assert_eq!(span.end, 5);
75        Ok(())
76    }
77
78    #[test]
79    fn token_span_try_new_end_before_start_errors() {
80        assert_eq!(
81            TokenSpan::try_new(10, 5),
82            Err(TokenSpanError::EndBeforeStart { start: 10, end: 5 })
83        );
84    }
85
86    #[test]
87    fn token_span_error_display_end_before_start() {
88        let err = TokenSpanError::EndBeforeStart { start: 10, end: 5 };
89        let msg = err.to_string();
90        assert!(msg.contains("10"));
91        assert!(msg.contains("5"));
92    }
93
94    #[test]
95    fn token_span_error_display_empty_span_not_allowed() {
96        let err = TokenSpanError::EmptySpanNotAllowed { kind: TokenKind::Identifier, at: 7 };
97        let msg = err.to_string();
98        assert!(msg.contains("Identifier"));
99        assert!(msg.contains("7"));
100    }
101
102    // --- Token ---
103
104    #[test]
105    fn token_new_stores_fields() {
106        let tok = Token::new(TokenKind::My, "my", 0, 2);
107        assert_eq!(tok.kind, TokenKind::My);
108        assert_eq!(&*tok.text, "my");
109        assert_eq!(tok.start, 0);
110        assert_eq!(tok.end, 2);
111    }
112
113    #[test]
114    fn token_len_saturates_for_inverted_span() {
115        let tok = Token::new(TokenKind::Identifier, "x", 9, 4);
116        assert_eq!(tok.len(), 0);
117        assert!(tok.is_empty());
118    }
119
120    #[test]
121    fn token_len_and_is_empty() {
122        let tok = Token::new(TokenKind::Identifier, "foo", 10, 13);
123        assert_eq!(tok.len(), 3);
124        assert!(!tok.is_empty());
125
126        let eof = Token::eof_at(8);
127        assert_eq!(eof.len(), 0);
128        assert!(eof.is_empty());
129    }
130
131    #[test]
132    fn token_span_and_range() {
133        let tok = Token::new(TokenKind::Number, "42", 5, 7);
134        assert_eq!(tok.span(), TokenSpan::new(5, 7));
135        assert_eq!(tok.range(), 5..7);
136    }
137
138    #[test]
139    fn token_try_new_allows_ordered_spans() -> Result<(), TokenSpanError> {
140        let tok = Token::try_new(TokenKind::Identifier, "name", 4, 8)?;
141        assert_eq!(tok.kind, TokenKind::Identifier);
142        assert_eq!(&*tok.text, "name");
143        assert_eq!(tok.span(), TokenSpan::new(4, 8));
144        Ok(())
145    }
146
147    #[test]
148    fn token_try_new_rejects_end_before_start() {
149        assert_eq!(
150            Token::try_new(TokenKind::Identifier, "x", 10, 5),
151            Err(TokenSpanError::EndBeforeStart { start: 10, end: 5 })
152        );
153    }
154
155    #[test]
156    fn token_new_checked_rejects_empty_non_eof() {
157        assert_eq!(
158            Token::new_checked(TokenKind::Identifier, "", 5, 5),
159            Err(TokenSpanError::EmptySpanNotAllowed { kind: TokenKind::Identifier, at: 5 })
160        );
161    }
162
163    #[test]
164    fn token_new_checked_allows_empty_eof() -> Result<(), TokenSpanError> {
165        let tok = Token::new_checked(TokenKind::Eof, "", 5, 5)?;
166        assert_eq!(tok.kind, TokenKind::Eof);
167        assert_eq!(tok.start, 5);
168        Ok(())
169    }
170
171    #[test]
172    fn token_new_checked_allows_empty_unknown() -> Result<(), TokenSpanError> {
173        let tok = Token::new_checked(TokenKind::Unknown, "", 6, 6)?;
174        assert_eq!(tok.kind, TokenKind::Unknown);
175        assert_eq!(tok.start, 6);
176        assert!(tok.is_empty());
177        Ok(())
178    }
179
180    #[test]
181    fn token_eof_at() {
182        let eof = Token::eof_at(42);
183        assert_eq!(eof.kind, TokenKind::Eof);
184        assert_eq!(eof.start, 42);
185        assert_eq!(eof.end, 42);
186        assert!(eof.is_empty());
187    }
188
189    #[test]
190    fn token_unknown_at_normalises_inverted_span() {
191        let tok = Token::unknown_at("?", 5, 3); // end < start
192        assert_eq!(tok.kind, TokenKind::Unknown);
193        assert_eq!(tok.start, 5);
194        assert_eq!(tok.end, 5); // bounded to start
195    }
196
197    #[test]
198    fn token_with_kind() {
199        let tok = Token::new(TokenKind::Identifier, "sub", 0, 3);
200        let retyped = tok.with_kind(TokenKind::Sub);
201        assert_eq!(retyped.kind, TokenKind::Sub);
202        assert_eq!(&*retyped.text, "sub");
203        assert_eq!(retyped.start, 0);
204        assert_eq!(retyped.end, 3);
205    }
206
207    #[test]
208    fn token_with_span_ok() -> Result<(), TokenSpanError> {
209        let tok = Token::new(TokenKind::String, "hello", 0, 5);
210        let moved = tok.with_span(10, 15)?;
211        assert_eq!(moved.start, 10);
212        assert_eq!(moved.end, 15);
213        Ok(())
214    }
215
216    #[test]
217    fn token_with_span_rejects_empty_non_eof() {
218        let tok = Token::new(TokenKind::String, "hello", 0, 5);
219        assert_eq!(
220            tok.with_span(10, 10),
221            Err(TokenSpanError::EmptySpanNotAllowed { kind: TokenKind::String, at: 10 })
222        );
223    }
224
225    #[test]
226    fn token_display_name_delegates_to_kind() {
227        let tok = Token::new(TokenKind::LeftBrace, "{", 0, 1);
228        assert_eq!(tok.display_name(), "'{'");
229    }
230
231    #[test]
232    fn token_as_ref_token_round_trip() {
233        let tok = Token::new(TokenKind::Sub, "sub", 0, 3);
234        let tok_ref = tok.as_ref_token();
235        assert_eq!(tok_ref.kind, TokenKind::Sub);
236        assert_eq!(tok_ref.text, "sub");
237        assert_eq!(tok_ref.start, 0);
238        assert_eq!(tok_ref.end, 3);
239
240        let owned: Token = tok_ref.into();
241        assert_eq!(owned.kind, TokenKind::Sub);
242        assert_eq!(&*owned.text, "sub");
243    }
244
245    // --- TokenRef ---
246
247    #[test]
248    fn token_ref_len_saturates_for_inverted_span() {
249        let r = TokenRef::new(TokenKind::Identifier, "x", 9, 4);
250        assert_eq!(r.len(), 0);
251        assert!(r.is_empty());
252    }
253
254    #[test]
255    fn token_ref_accessors() {
256        let r = TokenRef::new(TokenKind::Number, "99", 4, 6);
257        assert_eq!(r.len(), 2);
258        assert!(!r.is_empty());
259        assert_eq!(r.span(), (4, 6));
260        assert_eq!(r.display_name(), "number");
261    }
262
263    #[test]
264    fn token_ref_try_new_allows_ordered_spans() -> Result<(), TokenSpanError> {
265        let r = TokenRef::try_new(TokenKind::Number, "99", 4, 6)?;
266        assert_eq!(r.kind, TokenKind::Number);
267        assert_eq!(r.text, "99");
268        assert_eq!(r.span(), (4, 6));
269        Ok(())
270    }
271
272    #[test]
273    fn token_ref_to_owned_token() {
274        let r = TokenRef::new(TokenKind::Identifier, "foo", 1, 4);
275        let owned = r.to_owned_token();
276        assert_eq!(owned.kind, TokenKind::Identifier);
277        assert_eq!(&*owned.text, "foo");
278    }
279
280    // --- TokenKind::from_keyword ---
281
282    #[test]
283    fn from_keyword_recognises_perl_keywords() {
284        assert_eq!(TokenKind::from_keyword("my"), Some(TokenKind::My));
285        assert_eq!(TokenKind::from_keyword("sub"), Some(TokenKind::Sub));
286        assert_eq!(TokenKind::from_keyword("if"), Some(TokenKind::If));
287        assert_eq!(TokenKind::from_keyword("elsif"), Some(TokenKind::Elsif));
288        assert_eq!(TokenKind::from_keyword("else"), Some(TokenKind::Else));
289        assert_eq!(TokenKind::from_keyword("while"), Some(TokenKind::While));
290        assert_eq!(TokenKind::from_keyword("for"), Some(TokenKind::For));
291        assert_eq!(TokenKind::from_keyword("foreach"), Some(TokenKind::Foreach));
292        assert_eq!(TokenKind::from_keyword("return"), Some(TokenKind::Return));
293        assert_eq!(TokenKind::from_keyword("package"), Some(TokenKind::Package));
294        assert_eq!(TokenKind::from_keyword("use"), Some(TokenKind::Use));
295        assert_eq!(TokenKind::from_keyword("BEGIN"), Some(TokenKind::Begin));
296        assert_eq!(TokenKind::from_keyword("END"), Some(TokenKind::End));
297        assert_eq!(TokenKind::from_keyword("eval"), Some(TokenKind::Eval));
298        assert_eq!(TokenKind::from_keyword("class"), Some(TokenKind::Class));
299        assert_eq!(TokenKind::from_keyword("defer"), Some(TokenKind::Defer));
300        assert_eq!(TokenKind::from_keyword("and"), Some(TokenKind::WordAnd));
301        assert_eq!(TokenKind::from_keyword("or"), Some(TokenKind::WordOr));
302        assert_eq!(TokenKind::from_keyword("not"), Some(TokenKind::WordNot));
303        assert_eq!(TokenKind::from_keyword("xor"), Some(TokenKind::WordXor));
304        assert_eq!(TokenKind::from_keyword("cmp"), Some(TokenKind::StringCompare));
305    }
306
307    #[test]
308    fn from_keyword_unknown_returns_none() {
309        assert_eq!(TokenKind::from_keyword("MY"), None);
310        assert_eq!(TokenKind::from_keyword("Sub"), None);
311        assert_eq!(TokenKind::from_keyword("unknown"), None);
312        assert_eq!(TokenKind::from_keyword(""), None);
313    }
314
315    // --- TokenKind::from_operator ---
316
317    #[test]
318    fn from_operator_recognises_operators() {
319        assert_eq!(TokenKind::from_operator("="), Some(TokenKind::Assign));
320        assert_eq!(TokenKind::from_operator("+"), Some(TokenKind::Plus));
321        assert_eq!(TokenKind::from_operator("**"), Some(TokenKind::Power));
322        assert_eq!(TokenKind::from_operator("->"), Some(TokenKind::Arrow));
323        assert_eq!(TokenKind::from_operator("=>"), Some(TokenKind::FatArrow));
324        assert_eq!(TokenKind::from_operator("<=>"), Some(TokenKind::Spaceship));
325        assert_eq!(TokenKind::from_operator("//="), Some(TokenKind::DefinedOrAssign));
326        assert_eq!(TokenKind::from_operator("..."), Some(TokenKind::Ellipsis));
327        assert_eq!(TokenKind::from_operator("~~"), Some(TokenKind::SmartMatch));
328    }
329
330    #[test]
331    fn from_operator_unknown_returns_none() {
332        assert_eq!(TokenKind::from_operator(""), None);
333        assert_eq!(TokenKind::from_operator("xyz"), None);
334    }
335
336    // --- TokenKind::from_delimiter ---
337
338    #[test]
339    fn from_delimiter_recognises_all() {
340        assert_eq!(TokenKind::from_delimiter("("), Some(TokenKind::LeftParen));
341        assert_eq!(TokenKind::from_delimiter(")"), Some(TokenKind::RightParen));
342        assert_eq!(TokenKind::from_delimiter("{"), Some(TokenKind::LeftBrace));
343        assert_eq!(TokenKind::from_delimiter("}"), Some(TokenKind::RightBrace));
344        assert_eq!(TokenKind::from_delimiter("["), Some(TokenKind::LeftBracket));
345        assert_eq!(TokenKind::from_delimiter("]"), Some(TokenKind::RightBracket));
346        assert_eq!(TokenKind::from_delimiter(";"), Some(TokenKind::Semicolon));
347        assert_eq!(TokenKind::from_delimiter(","), Some(TokenKind::Comma));
348        assert_eq!(TokenKind::from_delimiter("x"), None);
349    }
350
351    // --- TokenKind::from_sigil ---
352
353    #[test]
354    fn from_sigil_recognises_all() {
355        assert_eq!(TokenKind::from_sigil("$"), Some(TokenKind::ScalarSigil));
356        assert_eq!(TokenKind::from_sigil("@"), Some(TokenKind::ArraySigil));
357        assert_eq!(TokenKind::from_sigil("%"), Some(TokenKind::HashSigil));
358        assert_eq!(TokenKind::from_sigil("&"), Some(TokenKind::SubSigil));
359        assert_eq!(TokenKind::from_sigil("*"), Some(TokenKind::GlobSigil));
360        assert_eq!(TokenKind::from_sigil("!"), None);
361    }
362
363    // --- TokenKind::category ---
364
365    #[test]
366    fn category_keyword_variants() {
367        assert_eq!(TokenKind::My.category(), TokenCategory::Keyword);
368        assert_eq!(TokenKind::Sub.category(), TokenCategory::Keyword);
369        assert_eq!(TokenKind::Defer.category(), TokenCategory::Keyword);
370    }
371
372    #[test]
373    fn category_operator_variants() {
374        assert_eq!(TokenKind::Plus.category(), TokenCategory::Operator);
375        assert_eq!(TokenKind::Spaceship.category(), TokenCategory::Operator);
376        assert_eq!(TokenKind::WordAnd.category(), TokenCategory::Operator);
377    }
378
379    #[test]
380    fn category_delimiter_variants() {
381        assert_eq!(TokenKind::LeftParen.category(), TokenCategory::Delimiter);
382        assert_eq!(TokenKind::Comma.category(), TokenCategory::Delimiter);
383    }
384
385    #[test]
386    fn category_literal_variants() {
387        assert_eq!(TokenKind::Number.category(), TokenCategory::Literal);
388        assert_eq!(TokenKind::HeredocStart.category(), TokenCategory::Literal);
389        assert_eq!(TokenKind::DataMarker.category(), TokenCategory::Literal);
390    }
391
392    #[test]
393    fn category_identifier_variants() {
394        assert_eq!(TokenKind::Identifier.category(), TokenCategory::Identifier);
395        assert_eq!(TokenKind::ScalarSigil.category(), TokenCategory::Identifier);
396        assert_eq!(TokenKind::GlobSigil.category(), TokenCategory::Identifier);
397    }
398
399    #[test]
400    fn category_special_variants() {
401        assert_eq!(TokenKind::Eof.category(), TokenCategory::Special);
402        assert_eq!(TokenKind::Unknown.category(), TokenCategory::Special);
403    }
404
405    // --- TokenKind::display_name ---
406
407    #[test]
408    fn display_name_selected_variants() {
409        assert_eq!(TokenKind::LeftBrace.display_name(), "'{'");
410        assert_eq!(TokenKind::RightBrace.display_name(), "'}'");
411        assert_eq!(TokenKind::Identifier.display_name(), "identifier");
412        assert_eq!(TokenKind::Eof.display_name(), "end of input");
413        assert_eq!(TokenKind::Number.display_name(), "number");
414        assert_eq!(TokenKind::Sub.display_name(), "'sub'");
415        assert_eq!(TokenKind::Semicolon.display_name(), "';'");
416        assert_eq!(TokenKind::HeredocStart.display_name(), "heredoc (<<)");
417        assert_eq!(TokenKind::DataMarker.display_name(), "data marker (__DATA__ or __END__)");
418    }
419
420    // --- TokenKind::all / metadata_count ---
421
422    #[test]
423    fn all_returns_132_variants() {
424        assert_eq!(TokenKind::all().len(), 132);
425        assert_eq!(TokenKind::metadata_count(), 132);
426    }
427
428    #[test]
429    fn metadata_round_trips_through_kind() {
430        let m = TokenKind::Sub.metadata();
431        assert_eq!(m.category, TokenCategory::Keyword);
432        assert_eq!(m.display_name, "'sub'");
433    }
434
435    // --- TokenKind role predicates ---
436
437    #[test]
438    fn is_assignment_operator_returns_true_for_assign_variants() {
439        assert!(TokenKind::Assign.is_assignment_operator());
440        assert!(TokenKind::PlusAssign.is_assignment_operator());
441        assert!(TokenKind::MinusAssign.is_assignment_operator());
442        assert!(TokenKind::StarAssign.is_assignment_operator());
443        assert!(TokenKind::SlashAssign.is_assignment_operator());
444        assert!(TokenKind::PercentAssign.is_assignment_operator());
445        assert!(TokenKind::DotAssign.is_assignment_operator());
446        assert!(TokenKind::AndAssign.is_assignment_operator());
447        assert!(TokenKind::OrAssign.is_assignment_operator());
448        assert!(TokenKind::XorAssign.is_assignment_operator());
449        assert!(TokenKind::PowerAssign.is_assignment_operator());
450        assert!(TokenKind::LeftShiftAssign.is_assignment_operator());
451        assert!(TokenKind::RightShiftAssign.is_assignment_operator());
452        assert!(TokenKind::LogicalAndAssign.is_assignment_operator());
453        assert!(TokenKind::LogicalOrAssign.is_assignment_operator());
454        assert!(TokenKind::DefinedOrAssign.is_assignment_operator());
455    }
456
457    #[test]
458    fn is_assignment_operator_returns_false_for_non_assign() {
459        assert!(!TokenKind::Plus.is_assignment_operator());
460        assert!(!TokenKind::Equal.is_assignment_operator());
461        assert!(!TokenKind::Identifier.is_assignment_operator());
462    }
463
464    #[test]
465    fn is_logical_operator_returns_true_for_logical_variants() {
466        assert!(TokenKind::And.is_logical_operator());
467        assert!(TokenKind::Or.is_logical_operator());
468        assert!(TokenKind::Not.is_logical_operator());
469        assert!(TokenKind::DefinedOr.is_logical_operator());
470        assert!(TokenKind::WordAnd.is_logical_operator());
471        assert!(TokenKind::WordOr.is_logical_operator());
472        assert!(TokenKind::WordNot.is_logical_operator());
473        assert!(TokenKind::WordXor.is_logical_operator());
474    }
475
476    #[test]
477    fn is_logical_operator_returns_false_for_non_logical() {
478        assert!(!TokenKind::Plus.is_logical_operator());
479        assert!(!TokenKind::Assign.is_logical_operator());
480        assert!(!TokenKind::Identifier.is_logical_operator());
481    }
482
483    #[test]
484    fn is_open_delimiter_returns_true_for_open_delimiters() {
485        assert!(TokenKind::LeftParen.is_open_delimiter());
486        assert!(TokenKind::LeftBrace.is_open_delimiter());
487        assert!(TokenKind::LeftBracket.is_open_delimiter());
488    }
489
490    #[test]
491    fn is_open_delimiter_returns_false_for_non_open() {
492        assert!(!TokenKind::RightParen.is_open_delimiter());
493        assert!(!TokenKind::Semicolon.is_open_delimiter());
494        assert!(!TokenKind::Plus.is_open_delimiter());
495    }
496
497    #[test]
498    fn is_quote_like_returns_true_for_quote_variants() {
499        assert!(TokenKind::Regex.is_quote_like());
500        assert!(TokenKind::Substitution.is_quote_like());
501        assert!(TokenKind::Transliteration.is_quote_like());
502        assert!(TokenKind::QuoteSingle.is_quote_like());
503        assert!(TokenKind::QuoteDouble.is_quote_like());
504        assert!(TokenKind::QuoteWords.is_quote_like());
505        assert!(TokenKind::QuoteCommand.is_quote_like());
506        assert!(TokenKind::HeredocStart.is_quote_like());
507    }
508
509    #[test]
510    fn is_quote_like_returns_false_for_non_quote() {
511        assert!(!TokenKind::String.is_quote_like());
512        assert!(!TokenKind::Identifier.is_quote_like());
513        assert!(!TokenKind::LeftParen.is_quote_like());
514    }
515
516    #[test]
517    fn is_recovery_boundary_returns_true_for_boundaries() {
518        assert!(TokenKind::Semicolon.is_recovery_boundary());
519        assert!(TokenKind::RightParen.is_recovery_boundary());
520        assert!(TokenKind::RightBrace.is_recovery_boundary());
521        assert!(TokenKind::RightBracket.is_recovery_boundary());
522        assert!(TokenKind::Eof.is_recovery_boundary());
523    }
524
525    #[test]
526    fn is_recovery_boundary_returns_false_for_non_boundary() {
527        assert!(!TokenKind::Plus.is_recovery_boundary());
528        assert!(!TokenKind::Identifier.is_recovery_boundary());
529        assert!(!TokenKind::LeftParen.is_recovery_boundary());
530    }
531
532    // --- TokenRef::new_checked branches ---
533
534    #[test]
535    fn token_ref_new_checked_rejects_end_before_start() {
536        assert_eq!(
537            TokenRef::new_checked(TokenKind::Identifier, "x", 10, 3),
538            Err(TokenSpanError::EndBeforeStart { start: 10, end: 3 })
539        );
540    }
541
542    #[test]
543    fn token_ref_new_checked_allows_empty_eof() -> Result<(), Box<dyn std::error::Error>> {
544        let tok = TokenRef::new_checked(TokenKind::Eof, "", 7, 7)?;
545        assert_eq!(tok.kind, TokenKind::Eof);
546        assert_eq!(tok.start, 7);
547        assert!(tok.is_empty());
548        Ok(())
549    }
550
551    #[test]
552    fn token_ref_new_checked_allows_empty_unknown() -> Result<(), Box<dyn std::error::Error>> {
553        let tok = TokenRef::new_checked(TokenKind::Unknown, "", 3, 3)?;
554        assert_eq!(tok.kind, TokenKind::Unknown);
555        assert_eq!(tok.start, 3);
556        assert!(tok.is_empty());
557        Ok(())
558    }
559
560    #[test]
561    fn token_ref_new_checked_rejects_empty_non_eof() {
562        assert_eq!(
563            TokenRef::new_checked(TokenKind::Identifier, "", 5, 5),
564            Err(TokenSpanError::EmptySpanNotAllowed { kind: TokenKind::Identifier, at: 5 })
565        );
566    }
567}
perl_token/lib.rs

perl_token/
lib.rs
