stryke/
token.rs

1/// `Token` — see variants.
2#[derive(Debug, Clone, PartialEq)]
3pub enum Token {
4    // Literals
5    /// `Integer` variant.
6    Integer(i64),
7    /// `Float` variant.
8    Float(f64),
9    /// `SingleString` variant.
10    SingleString(String),
11    /// `DoubleString` variant.
12    DoubleString(String),
13    /// `` `...` `` or `qx{...}` — interpolated like double quotes, then executed as `sh -c` (Perl `qx`).
14    BacktickString(String),
15    /// Regex pattern: (pattern, flags, delimiter)
16    Regex(String, String, char),
17    /// `HereDoc` variant.
18    HereDoc(String, String, bool),
19    /// `QW` variant.
20    QW(Vec<String>),
21
22    // Variables
23    /// `ScalarVar` variant.
24    ScalarVar(String),
25    /// `$$foo` — symbolic scalar deref (inner name is `foo` without sigil).
26    DerefScalarVar(String),
27    /// `ArrayVar` variant.
28    ArrayVar(String),
29    /// `HashVar` variant.
30    HashVar(String),
31    /// `ArrayAt` variant.
32    ArrayAt,
33    /// `HashPercent` variant.
34    HashPercent,
35
36    // Identifiers & keywords
37    /// `Ident` variant.
38    Ident(String),
39    /// `Label` variant.
40    Label(String),
41    /// `PackageSep` variant.
42    PackageSep,
43    /// `format NAME =` … body … `.` (body lines without the closing `.`)
44    FormatDecl {
45        name: String,
46        lines: Vec<String>,
47    },
48
49    // Arithmetic
50    /// `Plus` variant.
51    Plus,
52    /// `Minus` variant.
53    Minus,
54    /// `Star` variant.
55    Star,
56    /// `Slash` variant.
57    Slash,
58    /// `Percent` variant.
59    Percent,
60    /// `Power` variant.
61    Power,
62
63    // String
64    /// `Dot` variant.
65    Dot,
66    /// `X` variant.
67    X,
68
69    // Comparison (numeric)
70    /// `NumEq` variant.
71    NumEq,
72    /// `NumNe` variant.
73    NumNe,
74    /// `NumLt` variant.
75    NumLt,
76    /// `NumGt` variant.
77    NumGt,
78    /// `NumLe` variant.
79    NumLe,
80    /// `NumGe` variant.
81    NumGe,
82    /// `Spaceship` variant.
83    Spaceship,
84
85    // Comparison (string)
86    /// `StrEq` variant.
87    StrEq,
88    /// `StrNe` variant.
89    StrNe,
90    /// `StrLt` variant.
91    StrLt,
92    /// `StrGt` variant.
93    StrGt,
94    /// `StrLe` variant.
95    StrLe,
96    /// `StrGe` variant.
97    StrGe,
98    /// `StrCmp` variant.
99    StrCmp,
100
101    // Logical
102    /// `LogAnd` variant.
103    LogAnd,
104    /// `LogOr` variant.
105    LogOr,
106    /// `LogNot` variant.
107    LogNot,
108    /// `LogAndWord` variant.
109    LogAndWord,
110    /// `LogOrWord` variant.
111    LogOrWord,
112    /// `LogNotWord` variant.
113    LogNotWord,
114    /// `DefinedOr` variant.
115    DefinedOr,
116
117    // Bitwise
118    /// `BitAnd` variant.
119    BitAnd,
120    /// `BitOr` variant.
121    BitOr,
122    /// `BitXor` variant.
123    BitXor,
124    /// `BitNot` variant.
125    BitNot,
126    /// `ShiftLeft` variant.
127    ShiftLeft,
128    /// `ShiftRight` variant.
129    ShiftRight,
130
131    // Assignment
132    /// `Assign` variant.
133    Assign,
134    /// `PlusAssign` variant.
135    PlusAssign,
136    /// `MinusAssign` variant.
137    MinusAssign,
138    /// `MulAssign` variant.
139    MulAssign,
140    /// `DivAssign` variant.
141    DivAssign,
142    /// `ModAssign` variant.
143    ModAssign,
144    /// `PowAssign` variant.
145    PowAssign,
146    /// `DotAssign` variant.
147    DotAssign,
148    /// `x=` — string-repetition compound assign (`$s x= 3`).
149    XAssign,
150    /// `AndAssign` variant.
151    AndAssign,
152    /// `OrAssign` variant.
153    OrAssign,
154    /// `XorAssign` variant.
155    XorAssign,
156    /// `ShiftLeftAssign` variant.
157    ShiftLeftAssign,
158    /// `ShiftRightAssign` variant.
159    ShiftRightAssign,
160    /// Bitwise `&=`
161    BitAndAssign,
162    /// Bitwise `|=`
163    BitOrAssign,
164    /// `DefinedOrAssign` variant.
165    DefinedOrAssign,
166
167    // Increment/Decrement
168    /// `Increment` variant.
169    Increment,
170    /// `Decrement` variant.
171    Decrement,
172
173    // Regex binding
174    /// `BindMatch` variant.
175    BindMatch,
176    /// `BindNotMatch` variant.
177    BindNotMatch,
178
179    // Arrows & separators
180    /// `Arrow` variant.
181    Arrow,
182    /// `FatArrow` variant.
183    FatArrow,
184    /// `|>` — pipe-forward (F#/Elixir): `x |> f(a)` desugars to `f(x, a)` at parse time.
185    PipeForward,
186    /// `~>` — thread-first macro: `~> EXPR stage1 stage2 ...` injects as first arg
187    ThreadArrow,
188    /// `~>>` / `->>` — thread-last macro: injects as last arg
189    ThreadArrowLast,
190    /// `~s>` — streaming thread-first. Per-stage semantics match `~>`
191    /// (insert threaded value as first arg / topic), but each stage runs
192    /// in its own worker connected by bounded channels — items flow one
193    /// at a time. Concurrent (per-item flow with backpressure), not
194    /// chunk-parallel.
195    ThreadArrowStream,
196    /// `~s>>` — streaming thread-last. Per-stage semantics match `~>>`
197    /// (insert threaded value as last arg).
198    ThreadArrowStreamLast,
199    /// `~p>` — parallel-chunk thread-first. Whole pipeline runs per chunk
200    /// in parallel, results auto-merged at end (sugar for
201    /// `par_reduce { stage1 |> stage2 |> ... } SOURCE`). `||>` or
202    /// `|then|` switch from parallel-chunk back to pipe-forward / `~>`.
203    ThreadArrowPar,
204    /// `~p>>` — parallel-chunk thread-last counterpart of `~p>`.
205    ThreadArrowParLast,
206    /// `~d>` — **distributed** thread-first. Same chunk-block semantics as
207    /// `~p>` (each stage operates on `@_` = chunk elements), but the chunks
208    /// are shipped to remote workers on a cluster instead of local rayon
209    /// threads. Syntax: `~d> on $cluster SOURCE stage1 stage2 ...`.
210    /// Sugar for `dist_reduce on $cluster { stages } SOURCE`. Reuses the
211    /// existing `pmap_on` dispatcher (one ssh process per slot, JOB frames
212    /// flowing over a shared work queue, fault tolerance via retry).
213    ThreadArrowDist,
214    /// `~d>>` — distributed thread-last counterpart of `~d>` (insert threaded
215    /// value as last positional arg to each named stage).
216    ThreadArrowDistLast,
217    /// Two-dot range / inclusive flip-flop (`..`).
218    Range,
219    /// Three-dot range / exclusive flip-flop (`...`); list expansion matches `..` (Perl).
220    RangeExclusive,
221    /// `Backslash` variant.
222    Backslash,
223
224    // Delimiters
225    /// `LParen` variant.
226    LParen,
227    /// `RParen` variant.
228    RParen,
229    /// `LBracket` variant.
230    LBracket,
231    /// `RBracket` variant.
232    RBracket,
233    /// `LBrace` variant.
234    LBrace,
235    /// `RBrace` variant.
236    RBrace,
237    /// `>{` — standalone block in thread macro (not attached to a function)
238    ArrowBrace,
239
240    // Punctuation
241    Semicolon,
242    Comma,
243    Question,
244    Colon,
245
246    // I/O
247    Diamond,
248    ReadLine(String),
249
250    // File tests
251    FileTest(char),
252
253    // Special
254    Eof,
255    Newline,
256}
257
258impl Token {
259    /// `is_term_start` — see implementation.
260    pub fn is_term_start(&self) -> bool {
261        matches!(
262            self,
263            Token::Integer(_)
264                | Token::Float(_)
265                | Token::SingleString(_)
266                | Token::DoubleString(_)
267                | Token::BacktickString(_)
268                | Token::ScalarVar(_)
269                | Token::DerefScalarVar(_)
270                | Token::ArrayVar(_)
271                | Token::HashVar(_)
272                | Token::Ident(_)
273                | Token::LParen
274                | Token::LBracket
275                | Token::LBrace
276                | Token::Backslash
277                | Token::Minus
278                | Token::LogNot
279                | Token::BitNot
280                | Token::LogNotWord
281                | Token::QW(_)
282                | Token::Regex(_, _, _)
283                | Token::FileTest(_)
284                | Token::ThreadArrow
285                | Token::ThreadArrowLast
286                | Token::ThreadArrowStream
287                | Token::ThreadArrowStreamLast
288                | Token::ThreadArrowPar
289                | Token::ThreadArrowParLast
290        )
291    }
292}
293
294/// Decl-keyword classifier — central source of truth for the
295/// `my` / `var` / `val` / `mysync` / `varsync` family.
296///
297/// These five keywords all introduce a NEW binding (block-scoped lexical
298/// or sync-aware) and the parser/compiler/lsp historically open-coded
299/// `matches!(s, "my" | "var" | …)` in dozens of places. Centralizing here
300/// means a future keyword addition (e.g. `letsync`) flips one bit in one
301/// function instead of needing to chase down every hardcoded literal.
302///
303/// `our` / `oursync` / `local` / `state` are intentionally excluded —
304/// they have distinct semantics and the parser routes them through
305/// separate StmtKinds.
306#[inline]
307pub fn is_declaration(s: &str) -> bool {
308    matches!(s, "my" | "var" | "val" | "mysync" | "varsync")
309}
310
311/// Lexical (block-scoped) decl subset: `my` / `var` / `val`. All three
312/// route through `StmtKind::My`; `val` additionally marks the binding
313/// frozen at compile time.
314#[inline]
315pub fn is_lexical_decl(s: &str) -> bool {
316    matches!(s, "my" | "var" | "val")
317}
318
319/// Lexical decl OR package-level `our`. Used in places that accept
320/// either form (e.g. for-loop iterator decl).
321#[inline]
322pub fn is_lexical_or_our_decl(s: &str) -> bool {
323    is_lexical_decl(s) || s == "our"
324}
325
326/// Modern (post-style) decl subset: `var` / `val`. Distinguished from
327/// classic Perl `my` in a few parser sites (e.g. the for-loop init
328/// where `var`/`val` are accepted in modifier position).
329#[inline]
330pub fn is_post_style_decl(s: &str) -> bool {
331    matches!(s, "var" | "val")
332}
333
334/// Parallel-aware decl: `mysync` / `varsync`. Each wraps the variable
335/// in the Rayon-shared machinery.
336#[inline]
337pub fn is_sync_decl(s: &str) -> bool {
338    matches!(s, "mysync" | "varsync")
339}
340
341/// `val` (the frozen / immutable variant). Compile-time enforced via
342/// `scope_stack.frozen_scalars`.
343#[inline]
344pub fn is_frozen_decl(s: &str) -> bool {
345    s == "val"
346}
347
348/// Resolve an identifier to a keyword token or leave as Ident.
349pub fn keyword_or_ident(word: &str) -> Token {
350    match word {
351        "x" => Token::X,
352        "eq" => Token::StrEq,
353        "ne" => Token::StrNe,
354        "lt" => Token::StrLt,
355        "gt" => Token::StrGt,
356        "le" => Token::StrLe,
357        "ge" => Token::StrGe,
358        "cmp" => Token::StrCmp,
359        "and" => Token::LogAndWord,
360        "or" => Token::LogOrWord,
361        "not" => Token::LogNotWord,
362        _ => Token::Ident(word.to_string()),
363    }
364}
365
366/// All Perl keyword identifiers that are NOT converted to separate token variants.
367/// The parser recognizes these as `Token::Ident("keyword")`.
368pub const KEYWORDS: &[&str] = &[
369    "frozen",
370    "typed",
371    "my",
372    "var",
373    "val",
374    "mysync",
375    // `varsync` — Kotlin-style declarator alias for `mysync`, same
376    // way `var` aliases `my`. Both spell the same lockless atomic
377    // shared-state binding at the AST level (StmtKind::MySync).
378    "varsync",
379    "our",
380    "oursync",
381    "local",
382    "sub",
383    "fn",
384    "struct",
385    "enum",
386    "class",
387    "trait",
388    "extends",
389    "impl",
390    "pub",
391    "priv",
392    "Self",
393    "return",
394    "if",
395    "elsif",
396    "else",
397    "unless",
398    "while",
399    "until",
400    // `loop { ... }` — Rust-style infinite loop, desugars to `while (1)`.
401    "loop",
402    // `ploop [N] { ... }` — parallel `loop`, desugars to `pfor { while (1) { … } } 1..N`.
403    "ploop",
404    // `pwhile [N] (COND) { ... }` — parallel `while`, same desugar with COND.
405    "pwhile",
406    "for",
407    "foreach",
408    "do",
409    "last",
410    "next",
411    "redo",
412    "use",
413    // `import` — alias for `use`. Parses identically (same `parse_use`
414    // path, same StmtKind variants). Lets Python/JS-shaped scripts spell
415    // `import Foo;` while keeping `use Foo;` working.
416    "import",
417    "no",
418    "require",
419    "package",
420    "bless",
421    "print",
422    "say",
423    "die",
424    "warn",
425    "chomp",
426    "chop",
427    "push",
428    "pop",
429    "shift",
430    "shuffle",
431    "chunked",
432    "windowed",
433    "unshift",
434    "splice",
435    "split",
436    "join",
437    "json_decode",
438    "json_encode",
439    "json_jq",
440    "jwt_decode",
441    "jwt_decode_unsafe",
442    "jwt_encode",
443    "log_debug",
444    "log_error",
445    "log_info",
446    "log_json",
447    "log_level",
448    "log_trace",
449    "log_warn",
450    "sha256",
451    "sha1",
452    "md5",
453    "hmac_sha256",
454    "hmac",
455    "uuid",
456    "base64_encode",
457    "base64_decode",
458    "hex_encode",
459    "hex_decode",
460    "gzip",
461    "gunzip",
462    "zstd",
463    "zstd_decode",
464    "datetime_utc",
465    "datetime_from_epoch",
466    "datetime_parse_rfc3339",
467    "datetime_strftime",
468    "toml_decode",
469    "toml_encode",
470    "yaml_decode",
471    "yaml_encode",
472    "url_encode",
473    "url_decode",
474    "uri_escape",
475    "uri_unescape",
476    "sort",
477    "reverse",
478    "reversed",
479    "map",
480    "maps",
481    "flat_map",
482    "flat_maps",
483    "flatten",
484    "compact",
485    "reject",
486    "grepv",
487    "concat",
488    "chain",
489    "set",
490    "list_count",
491    "list_size",
492    "count",
493    "size",
494    "cnt",
495    "inject",
496    "first",
497    "detect",
498    "find",
499    "find_all",
500    "match",
501    "grep",
502    "greps",
503    "keys",
504    "values",
505    "each",
506    "delete",
507    "exists",
508    "open",
509    "close",
510    "read",
511    "write",
512    "seek",
513    "tell",
514    "eof",
515    "defined",
516    "undef",
517    // `null` — stryke alias for `undef` (JS/SQL spelling).
518    "null",
519    "ref",
520    "eval",
521    "exec",
522    "system",
523    "chdir",
524    "mkdir",
525    "rmdir",
526    "unlink",
527    "rename",
528    "chmod",
529    "chown",
530    "length",
531    "substr",
532    "index",
533    "rindex",
534    "sprintf",
535    "printf",
536    "lc",
537    "uc",
538    "lcfirst",
539    "ucfirst",
540    "hex",
541    "oct",
542    "int",
543    "abs",
544    "sqrt",
545    "scalar",
546    "wantarray",
547    "caller",
548    "exit",
549    "pos",
550    "quotemeta",
551    "chr",
552    "ord",
553    "pack",
554    "unpack",
555    "vec",
556    "tie",
557    "untie",
558    "tied",
559    "chomp",
560    "chop",
561    "defined",
562    "dump",
563    "each",
564    "exists",
565    "formline",
566    "lock",
567    "prototype",
568    "reset",
569    "scalar",
570    "BEGIN",
571    "END",
572    "INIT",
573    "CHECK",
574    "UNITCHECK",
575    "AUTOLOAD",
576    "DESTROY",
577    "all",
578    "any",
579    "none",
580    "take_while",
581    "drop_while",
582    "skip_while",
583    "skip",
584    "first_or",
585    "tap",
586    "peek",
587    "with_index",
588    "pmap",
589    "pflat_map",
590    "puniq",
591    "pfirst",
592    "pany",
593    "pmap_chunked",
594    "pipeline",
595    "pgrep",
596    "pfor",
597    // `pforeach` — alias for `pfor`, same as `for`/`foreach`.
598    "pforeach",
599    "par_lines",
600    "par_walk",
601    "pwatch",
602    "psort",
603    "reduce",
604    "fold",
605    "preduce",
606    "preduce_init",
607    "pmap_reduce",
608    "pcache",
609    "watch",
610    "tie",
611    "fan",
612    "fan_cap",
613    "pchannel",
614    "pselect",
615    "uniq",
616    "distinct",
617    "uniqstr",
618    "uniqint",
619    "uniqnum",
620    "pairs",
621    "unpairs",
622    "pairkeys",
623    "pairvalues",
624    "pairgrep",
625    "pairmap",
626    "pairfirst",
627    "sample",
628    "zip",
629    "zip_shortest",
630    "mesh",
631    "mesh_shortest",
632    "notall",
633    "reductions",
634    "sum",
635    "sum0",
636    "product",
637    "min",
638    "max",
639    "minstr",
640    "maxstr",
641    "mean",
642    "median",
643    "mode",
644    "stddev",
645    "variance",
646    "async",
647    "spawn",
648    "trace",
649    "timer",
650    "bench",
651    "await",
652    "slurp",
653    "swallow",
654    "ingest",
655    "burp",
656    "god",
657    "capture",
658    "fetch_url",
659    "fetch",
660    "fetch_json",
661    "fetch_async",
662    "fetch_async_json",
663    "json_jq",
664    "par_fetch",
665    "par_pipeline",
666    "par_csv_read",
667    "par_sed",
668    "try",
669    "catch",
670    "finally",
671    "given",
672    "when",
673    "default",
674    "eval_timeout",
675    "thread",
676    "t",
677];
678
679#[cfg(test)]
680mod tests {
681    use super::*;
682
683    #[test]
684    fn keyword_or_ident_maps_string_ops() {
685        assert!(matches!(keyword_or_ident("eq"), Token::StrEq));
686        assert!(matches!(keyword_or_ident("cmp"), Token::StrCmp));
687    }
688
689    #[test]
690    fn keyword_or_ident_non_keyword_is_ident() {
691        assert!(matches!(
692            keyword_or_ident("foo_bar"),
693            Token::Ident(s) if s == "foo_bar"
694        ));
695    }
696
697    #[test]
698    fn keyword_or_ident_logical_words_and_repeat() {
699        assert!(matches!(keyword_or_ident("and"), Token::LogAndWord));
700        assert!(matches!(keyword_or_ident("or"), Token::LogOrWord));
701        assert!(matches!(keyword_or_ident("not"), Token::LogNotWord));
702        assert!(matches!(keyword_or_ident("x"), Token::X));
703    }
704
705    #[test]
706    fn keyword_or_ident_string_comparison_words() {
707        assert!(matches!(keyword_or_ident("lt"), Token::StrLt));
708        assert!(matches!(keyword_or_ident("gt"), Token::StrGt));
709        assert!(matches!(keyword_or_ident("ge"), Token::StrGe));
710    }
711
712    #[test]
713    fn keyword_or_ident_string_le_ne() {
714        assert!(matches!(keyword_or_ident("le"), Token::StrLe));
715        assert!(matches!(keyword_or_ident("ne"), Token::StrNe));
716    }
717
718    #[test]
719    fn keyword_or_ident_control_flow_keywords() {
720        assert!(matches!(keyword_or_ident("if"), Token::Ident(s) if s == "if"));
721        assert!(matches!(keyword_or_ident("else"), Token::Ident(s) if s == "else"));
722        assert!(matches!(keyword_or_ident("elsif"), Token::Ident(s) if s == "elsif"));
723        assert!(matches!(keyword_or_ident("unless"), Token::Ident(s) if s == "unless"));
724        assert!(matches!(keyword_or_ident("while"), Token::Ident(s) if s == "while"));
725        assert!(matches!(keyword_or_ident("until"), Token::Ident(s) if s == "until"));
726        assert!(matches!(keyword_or_ident("for"), Token::Ident(s) if s == "for"));
727        assert!(matches!(keyword_or_ident("foreach"), Token::Ident(s) if s == "foreach"));
728        assert!(matches!(keyword_or_ident("return"), Token::Ident(s) if s == "return"));
729    }
730
731    #[test]
732    fn keyword_or_ident_declarations() {
733        assert!(matches!(keyword_or_ident("my"), Token::Ident(s) if s == "my"));
734        assert!(matches!(keyword_or_ident("typed"), Token::Ident(s) if s == "typed"));
735        assert!(matches!(keyword_or_ident("our"), Token::Ident(s) if s == "our"));
736        assert!(matches!(keyword_or_ident("local"), Token::Ident(s) if s == "local"));
737        assert!(matches!(keyword_or_ident("sub"), Token::Ident(s) if s == "sub"));
738        assert!(matches!(keyword_or_ident("package"), Token::Ident(s) if s == "package"));
739    }
740
741    #[test]
742    fn keyword_or_ident_io_and_list_ops() {
743        assert!(matches!(keyword_or_ident("print"), Token::Ident(s) if s == "print"));
744        assert!(matches!(keyword_or_ident("say"), Token::Ident(s) if s == "say"));
745        assert!(matches!(keyword_or_ident("map"), Token::Ident(s) if s == "map"));
746        assert!(matches!(keyword_or_ident("grep"), Token::Ident(s) if s == "grep"));
747        assert!(matches!(keyword_or_ident("sort"), Token::Ident(s) if s == "sort"));
748        assert!(matches!(keyword_or_ident("join"), Token::Ident(s) if s == "join"));
749        assert!(matches!(keyword_or_ident("split"), Token::Ident(s) if s == "split"));
750        assert!(matches!(
751            keyword_or_ident("list_count"),
752            Token::Ident(s) if s == "list_count"
753        ));
754        assert!(matches!(
755            keyword_or_ident("list_size"),
756            Token::Ident(s) if s == "list_size"
757        ));
758        assert!(matches!(keyword_or_ident("cnt"), Token::Ident(s) if s == "cnt"));
759        assert!(matches!(
760            keyword_or_ident("capture"),
761            Token::Ident(s) if s == "capture"
762        ));
763    }
764
765    #[test]
766    fn keyword_or_ident_parallel_primitives() {
767        assert!(matches!(keyword_or_ident("pmap"), Token::Ident(s) if s == "pmap"));
768        assert!(matches!(
769            keyword_or_ident("pmap_chunked"),
770            Token::Ident(s) if s == "pmap_chunked"
771        ));
772        assert!(matches!(
773            keyword_or_ident("pipeline"),
774            Token::Ident(s) if s == "pipeline"
775        ));
776        assert!(matches!(keyword_or_ident("pgrep"), Token::Ident(s) if s == "pgrep"));
777        assert!(matches!(keyword_or_ident("pfor"), Token::Ident(s) if s == "pfor"));
778        assert!(matches!(keyword_or_ident("psort"), Token::Ident(s) if s == "psort"));
779        assert!(matches!(keyword_or_ident("reduce"), Token::Ident(s) if s == "reduce"));
780        assert!(matches!(keyword_or_ident("fold"), Token::Ident(s) if s == "fold"));
781        assert!(matches!(keyword_or_ident("preduce"), Token::Ident(s) if s == "preduce"));
782        assert!(matches!(keyword_or_ident("fan"), Token::Ident(s) if s == "fan"));
783        assert!(matches!(keyword_or_ident("trace"), Token::Ident(s) if s == "trace"));
784        assert!(matches!(keyword_or_ident("timer"), Token::Ident(s) if s == "timer"));
785    }
786
787    #[test]
788    fn keyword_or_ident_type_and_ref() {
789        assert!(matches!(keyword_or_ident("ref"), Token::Ident(s) if s == "ref"));
790        assert!(matches!(keyword_or_ident("scalar"), Token::Ident(s) if s == "scalar"));
791        assert!(matches!(keyword_or_ident("defined"), Token::Ident(s) if s == "defined"));
792        assert!(matches!(keyword_or_ident("undef"), Token::Ident(s) if s == "undef"));
793    }
794
795    #[test]
796    fn keyword_or_ident_block_hooks() {
797        assert!(matches!(keyword_or_ident("BEGIN"), Token::Ident(s) if s == "BEGIN"));
798        assert!(matches!(keyword_or_ident("END"), Token::Ident(s) if s == "END"));
799        assert!(matches!(keyword_or_ident("INIT"), Token::Ident(s) if s == "INIT"));
800    }
801
802    #[test]
803    fn keyword_or_ident_plain_identifier_untouched() {
804        assert!(matches!(
805            keyword_or_ident("xyzzy123"),
806            Token::Ident(s) if s == "xyzzy123"
807        ));
808    }
809}
stryke/token.rs

stryke/
token.rs
