oxc_parser/lexer/
token.rs

1//! Token
2
3use std::{fmt, mem, ptr::NonNull};
4
5use oxc_span::Span;
6
7use super::kind::Kind;
8
9// Bit layout for `u128`:
10// - Bits 0-31 (32 bits): `start` (`u32`)
11// - Bits 32-63 (32 bits): `end` (`u32`)
12// - Bits 64-71 (8 bits): `kind` (`Kind`)
13// - Bits 72-79 (8 bits): `is_on_new_line` (`bool`)
14// - Bits 80-87 (8 bits): `escaped` (`bool`)
15// - Bits 88-95 (8 bits): `lone_surrogates` (`bool`)
16// - Bits 96-103 (8 bits): `has_separator` (`bool`)
17// - Bits 104-127 (24 bits): unused
18
19const START_SHIFT: usize = 0;
20const END_SHIFT: usize = 32;
21const KIND_SHIFT: usize = 64;
22const IS_ON_NEW_LINE_SHIFT: usize = 72;
23const ESCAPED_SHIFT: usize = 80;
24const LONE_SURROGATES_SHIFT: usize = 88;
25const HAS_SEPARATOR_SHIFT: usize = 96;
26
27const START_MASK: u128 = 0xFFFF_FFFF; // 32 bits
28const END_MASK: u128 = 0xFFFF_FFFF; // 32 bits
29const KIND_MASK: u128 = 0xFF; // 8 bits
30#[expect(dead_code)]
31const BOOL_MASK: u128 = 0xFF; // 8 bits
32
33const _: () = {
34    const fn is_valid_shift<T>(shift: usize) -> bool {
35        let align_bits = align_of::<T>() * 8;
36        shift.is_multiple_of(align_bits) && shift < u128::BITS as usize
37    }
38
39    // Check `u32` fields are aligned on 32 and in bounds, so can be read/written via pointers
40    assert!(is_valid_shift::<u32>(START_SHIFT));
41    assert!(is_valid_shift::<u32>(END_SHIFT));
42
43    // Check `Kind` is 1 byte, and `KIND_SHIFT` is aligned on 8 and in bounds, so can be read/written via pointers
44    assert!(size_of::<Kind>() == 1);
45    assert!(align_of::<Kind>() == 1);
46    assert!(is_valid_shift::<Kind>(KIND_SHIFT));
47
48    // Check flags fields are aligned on 8 and in bounds, so can be read/written via pointers
49    assert!(is_valid_shift::<bool>(IS_ON_NEW_LINE_SHIFT));
50    assert!(is_valid_shift::<bool>(ESCAPED_SHIFT));
51    assert!(is_valid_shift::<bool>(LONE_SURROGATES_SHIFT));
52    assert!(is_valid_shift::<bool>(HAS_SEPARATOR_SHIFT));
53};
54
55#[derive(Clone, Copy)]
56#[repr(transparent)]
57pub struct Token(u128);
58
59impl Default for Token {
60    #[inline]
61    fn default() -> Self {
62        // `Kind::default()` is `Kind::Eof`. So `0` is equivalent to:
63        // start: 0,
64        // end: 0,
65        // kind: Kind::default(),
66        // is_on_new_line: false,
67        // escaped: false,
68        // lone_surrogates: false,
69        // has_separator: false,
70        const _: () = assert!(Kind::Eof as u8 == 0);
71        Self(0)
72    }
73}
74
75impl fmt::Debug for Token {
76    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
77        f.debug_struct("Token")
78            .field("kind", &self.kind())
79            .field("start", &self.start())
80            .field("end", &self.end())
81            .field("is_on_new_line", &self.is_on_new_line())
82            .field("escaped", &self.escaped())
83            .field("lone_surrogates", &self.lone_surrogates())
84            .field("has_separator", &self.has_separator())
85            .finish()
86    }
87}
88
89impl Token {
90    #[inline]
91    pub(super) fn new_on_new_line() -> Self {
92        // Start with a default token, then set the flag
93        let mut token = Self::default();
94        token.set_is_on_new_line(true);
95        token
96    }
97}
98
99// Getters and setters.
100//
101// Prior to Rust 1.95.0, `set` methods used safe bitwise operations.
102// This regressed heavily in Rust 1.95.0 due to an LLVM bug:
103// https://github.com/oxc-project/oxc/pull/21509
104// https://github.com/rust-lang/rust/issues/155422
105//
106// To obtain the same tight assembly as before on Rust 1.95.0, we now use unsafe pointer manipulation
107// to directly write the "fields" of `Token`.
108// The original implementations are kept in comments, in case we want to revert to them once the LLVM bug is fixed.
109impl Token {
110    #[inline]
111    pub fn span(&self) -> Span {
112        Span::new(self.start(), self.end())
113    }
114
115    // `set_span` is only exposed as public API when `mutate_tokens` feature is enabled.
116    // Otherwise, it is only accessible within `lexer` module.
117    #[cfg(feature = "mutate_tokens")]
118    #[inline]
119    pub fn set_span(&mut self, span: Span) {
120        self.set_span_impl(span);
121    }
122
123    #[cfg(not(feature = "mutate_tokens"))]
124    #[inline]
125    #[allow(dead_code, clippy::allow_attributes)]
126    pub(super) fn set_span(&mut self, span: Span) {
127        self.set_span_impl(span);
128    }
129
130    #[inline]
131    fn set_span_impl(&mut self, span: Span) {
132        // On little-endian systems, `start` and `end` fields in `Span` are in same order as in `Token`,
133        // so compiler boils this down to just a `u64` write of the `Span` into the first 8 bytes of the `Token`
134        // https://godbolt.org/z/bdY5ccad6
135        self.set_start(span.start);
136        self.set_end(span.end);
137    }
138
139    #[inline]
140    pub fn start(&self) -> u32 {
141        ((self.0 >> START_SHIFT) & START_MASK) as u32
142    }
143
144    #[inline]
145    pub(super) fn set_start(&mut self, start: u32) {
146        /*
147        // Original version. Perf regressed in Rust 1.95.0.
148        self.0 &= !(START_MASK << START_SHIFT); // Clear current `start` bits
149        self.0 |= u128::from(start) << START_SHIFT;
150        */
151
152        // SAFETY: `START_SHIFT` is a valid `u32` field position in `Token`
153        unsafe { self.write_u32(START_SHIFT, start) };
154    }
155
156    #[inline]
157    pub fn end(&self) -> u32 {
158        ((self.0 >> END_SHIFT) & END_MASK) as u32
159    }
160
161    #[inline]
162    pub(super) fn set_end(&mut self, end: u32) {
163        let start = self.start();
164        debug_assert!(end >= start, "Token end ({end}) cannot be less than start ({start})");
165
166        /*
167        // Original version. Perf regressed in Rust 1.95.0.
168        self.0 &= !(END_MASK << END_SHIFT); // Clear current `end` bits
169        self.0 |= u128::from(end) << END_SHIFT;
170        */
171
172        // SAFETY: `END_SHIFT` is a valid `u32` field position in `Token`
173        unsafe { self.write_u32(END_SHIFT, end) };
174    }
175
176    #[inline]
177    pub fn kind(&self) -> Kind {
178        // SAFETY: `Kind` is `#[repr(u8)]`. Only `Token::default` and `Token::set_kind` set these bits,
179        // and they set them to the `u8` value of an existing `Kind`.
180        // So transmuting these bits back to `Kind` must produce a valid `Kind`.
181        unsafe { mem::transmute::<u8, Kind>(((self.0 >> KIND_SHIFT) & KIND_MASK) as u8) }
182    }
183
184    // `set_kind` is only exposed as public API when `mutate_tokens` feature is enabled.
185    // Otherwise, it is only accessible within `lexer` module.
186    #[cfg(feature = "mutate_tokens")]
187    #[inline]
188    pub fn set_kind(&mut self, kind: Kind) {
189        self.set_kind_impl(kind);
190    }
191
192    #[cfg(not(feature = "mutate_tokens"))]
193    #[inline]
194    pub(super) fn set_kind(&mut self, kind: Kind) {
195        self.set_kind_impl(kind);
196    }
197
198    #[inline]
199    fn set_kind_impl(&mut self, kind: Kind) {
200        /*
201        // Original version. Perf regressed in Rust 1.95.0.
202        self.0 &= !(KIND_MASK << KIND_SHIFT); // Clear current `kind` bits
203        self.0 |= u128::from(kind as u8) << KIND_SHIFT;
204        */
205
206        const OFFSET: usize =
207            if cfg!(target_endian = "little") { KIND_SHIFT / 8 } else { 15 - (KIND_SHIFT / 8) };
208        // SAFETY: `Kind` is `#[repr(u8)]`, so writing one byte at `OFFSET` overwrites only the `kind` byte
209        // without touching adjacent fields. These bits always represent a valid `Kind`.
210        // `Token` is borrowed mutably, so the write is unaliased.
211        unsafe { *NonNull::from(self).cast::<Kind>().add(OFFSET).as_mut() = kind };
212    }
213
214    /// Checks if this token appears at the start of a new line.
215    ///
216    /// Returns `true` if the token was preceded by a line terminator during lexical analysis.
217    /// This information is crucial for automatic semicolon insertion (ASI) and other
218    /// JavaScript parsing rules that depend on line boundaries.
219    #[inline]
220    pub fn is_on_new_line(&self) -> bool {
221        // Use a pointer read rather than arithmetic as it produces less instructions.
222        // SAFETY: 8 bits starting at `IS_ON_NEW_LINE_SHIFT` are only set in `Token::default` and
223        // `Token::set_is_on_new_line`. Both only set these bits to 0 or 1, so valid to read as a `bool`.
224        unsafe { self.read_bool(IS_ON_NEW_LINE_SHIFT) }
225    }
226
227    #[inline]
228    pub(super) fn set_is_on_new_line(&mut self, value: bool) {
229        /*
230        // Original version. Perf regressed in Rust 1.95.0.
231        self.0 &= !(BOOL_MASK << IS_ON_NEW_LINE_SHIFT); // Clear current `is_on_new_line` bits
232        self.0 |= u128::from(value) << IS_ON_NEW_LINE_SHIFT;
233        */
234
235        // SAFETY: `IS_ON_NEW_LINE_SHIFT` is a valid `bool` field position in `Token`
236        unsafe { self.write_bool(IS_ON_NEW_LINE_SHIFT, value) };
237    }
238
239    #[inline]
240    pub fn escaped(&self) -> bool {
241        // Use a pointer read rather than arithmetic as it produces less instructions.
242        // SAFETY: 8 bits starting at `ESCAPED_SHIFT` are only set in `Token::default` and
243        // `Token::set_escaped`. Both only set these bits to 0 or 1, so valid to read as a `bool`.
244        unsafe { self.read_bool(ESCAPED_SHIFT) }
245    }
246
247    #[inline]
248    pub(super) fn set_escaped(&mut self, escaped: bool) {
249        /*
250        // Original version. Perf regressed in Rust 1.95.0.
251        self.0 &= !(BOOL_MASK << ESCAPED_SHIFT); // Clear current `escaped` bits
252        self.0 |= u128::from(escaped) << ESCAPED_SHIFT;
253        */
254
255        // SAFETY: `ESCAPED_SHIFT` is a valid `bool` field position in `Token`
256        unsafe { self.write_bool(ESCAPED_SHIFT, escaped) };
257    }
258
259    #[inline]
260    pub fn lone_surrogates(&self) -> bool {
261        // Use a pointer read rather than arithmetic as it produces less instructions.
262        // SAFETY: 8 bits starting at `LONE_SURROGATES_SHIFT` are only set in `Token::default` and
263        // `Token::set_lone_surrogates`. Both only set these bits to 0 or 1, so valid to read as a `bool`.
264        unsafe { self.read_bool(LONE_SURROGATES_SHIFT) }
265    }
266
267    #[inline]
268    pub(super) fn set_lone_surrogates(&mut self, value: bool) {
269        /*
270        // Original version. Perf regressed in Rust 1.95.0.
271        self.0 &= !(BOOL_MASK << LONE_SURROGATES_SHIFT); // Clear current `lone_surrogates` bits
272        self.0 |= u128::from(value) << LONE_SURROGATES_SHIFT;
273        */
274
275        // SAFETY: `LONE_SURROGATES_SHIFT` is a valid `bool` field position in `Token`
276        unsafe { self.write_bool(LONE_SURROGATES_SHIFT, value) };
277    }
278
279    #[inline]
280    pub fn has_separator(&self) -> bool {
281        // Use a pointer read rather than arithmetic as it produces less instructions.
282        // SAFETY: 8 bits starting at `HAS_SEPARATOR_SHIFT` are only set in `Token::default` and
283        // `Token::set_has_separator`. Both only set these bits to 0 or 1, so valid to read as a `bool`.
284        unsafe { self.read_bool(HAS_SEPARATOR_SHIFT) }
285    }
286
287    #[inline]
288    pub(super) fn set_has_separator(&mut self, value: bool) {
289        /*
290        // Original version. Perf regressed in Rust 1.95.0.
291        self.0 &= !(BOOL_MASK << HAS_SEPARATOR_SHIFT); // Clear current `has_separator` bits
292        self.0 |= u128::from(value) << HAS_SEPARATOR_SHIFT;
293        */
294
295        // SAFETY: `HAS_SEPARATOR_SHIFT` is a valid `bool` field position in `Token`
296        unsafe { self.write_bool(HAS_SEPARATOR_SHIFT, value) };
297    }
298
299    /// Read `bool` from 8 bits starting at bit position `shift`.
300    ///
301    /// # SAFETY
302    ///
303    /// `shift` must be the location of a valid boolean "field" in [`Token`]
304    /// e.g. `ESCAPED_SHIFT`. The caller must guarantee that the 8 bits at
305    /// `shift` contain only 0 or 1, making it safe to read as a `bool`.
306    ///
307    /// # Performance analysis
308    ///
309    /// This method uses unsafe pointer arithmetic to directly read a boolean value
310    /// from the token's 128-bit representation. This approach is deliberately chosen
311    /// for performance optimization on hot paths.
312    ///
313    /// This unsafe pointer arithmetic approach generates only 1 CPU instruction:
314    /// ```asm
315    /// movzx   eax, byte ptr [rdi + 9]  ; Load byte at offset
316    /// ```
317    ///
318    /// Compared to the safe bit-shift alternative:
319    /// ```ignore
320    /// (token.0 >> shift) & 1 != 0
321    /// ```
322    ///
323    /// ```asm
324    /// movzx   eax, byte ptr [rdi + 9]  ; Load byte at offset
325    /// and     al, 1                    ; Mask to lower bit only
326    /// ```
327    ///
328    /// <https://godbolt.org/z/7xxrP348P>
329    ///
330    /// This optimization was retained after careful benchmarking (see PR #13788),
331    /// where the single instruction difference on hot paths justified keeping
332    /// the unsafe implementation.
333    #[expect(clippy::inline_always)]
334    #[inline(always)] // So `shift` is statically known
335    unsafe fn read_bool(&self, shift: usize) -> bool {
336        // Byte offset depends on endianness of the system
337        let offset = if cfg!(target_endian = "little") { shift / 8 } else { 15 - (shift / 8) };
338        // SAFETY: Caller guarantees `shift` points to valid `bool`.
339        // This method borrows `Token`, so valid to read field via a reference - can't be aliased.
340        unsafe {
341            let field_ptr = NonNull::from_ref(self).cast::<bool>().add(offset);
342            debug_assert!(field_ptr.cast::<u8>().read() <= 1);
343            *field_ptr.as_ref()
344        }
345    }
346
347    /// Write `bool` to the 8 bits starting at bit position `shift`.
348    ///
349    /// # SAFETY
350    ///
351    /// `shift` must be the location of a valid boolean "field" in [`Token`] e.g. `ESCAPED_SHIFT`.
352    ///
353    /// # Performance analysis
354    ///
355    /// Writing the whole byte via a pointer avoids a read-modify-write of the underlying `u128`.
356    /// LLVM stopped folding the safe `self.0 &= !mask; self.0 |= val << shift` pattern
357    /// into a single byte store as of rustc 1.95.0 - see <https://github.com/rust-lang/rust/issues/155422>.
358    /// This implementation produces `mov byte ptr [rdi + N], sil` on both affected and unaffected Rust versions.
359    #[expect(clippy::inline_always)]
360    #[inline(always)] // So `shift` is statically known
361    unsafe fn write_bool(&mut self, shift: usize, value: bool) {
362        // Byte offset depends on endianness of the system
363        let offset = if cfg!(target_endian = "little") { shift / 8 } else { 15 - (shift / 8) };
364        // SAFETY: Caller guarantees `shift` points to a valid `bool` field.
365        // `Token` is borrowed mutably, so the write is unaliased.
366        // `as_mut` produces a `&mut bool` with `noalias` metadata for LLVM.
367        unsafe { *NonNull::from(self).cast::<bool>().add(offset).as_mut() = value };
368    }
369
370    /// Write `u32` to the 32 bits starting at bit position `shift`.
371    ///
372    /// # SAFETY
373    ///
374    /// `shift` must be the location of a valid `u32` "field" in [`Token`] i.e. `START_SHIFT` or `END_SHIFT`.
375    ///
376    /// # Performance analysis
377    ///
378    /// See `write_bool` - same story, but `mov dword ptr [rdi + N], esi`.
379    #[expect(clippy::inline_always)]
380    #[inline(always)] // So `shift` is statically known
381    unsafe fn write_u32(&mut self, shift: usize, value: u32) {
382        // `Token` is 16 bytes = 4 `u32`s wide. Offset in `u32` units depends on endianness.
383        let offset = if cfg!(target_endian = "little") { shift / 32 } else { 3 - (shift / 32) };
384        // SAFETY: Caller guarantees `shift` points to a valid `u32` field (`start` or `end`).
385        // `Token` is `#[repr(transparent)]` over `u128`, so casting `NonNull<Token>` to `NonNull<u32>`
386        // is going from stricter to looser alignment. `Token` is borrowed mutably, so the
387        // write is unaliased - `as_mut` produces a `&mut u32` with `noalias` metadata for LLVM.
388        unsafe { *NonNull::from(self).cast::<u32>().add(offset).as_mut() = value };
389    }
390}
391
392#[cfg(test)]
393mod test {
394    use super::{Kind, Span, Token};
395
396    // Test size of `Token`
397    const _: () = assert!(size_of::<Token>() == 16);
398
399    // Test default token values
400    #[test]
401    fn default_token_values() {
402        let token = Token::default();
403        assert_eq!(token.start(), 0);
404        assert_eq!(token.end(), 0);
405        assert_eq!(token.kind(), Kind::Eof); // Kind::default() is Eof
406        assert!(!token.is_on_new_line());
407        assert!(!token.escaped());
408        assert!(!token.lone_surrogates());
409        assert!(!token.has_separator());
410    }
411
412    #[test]
413    fn new_on_new_line_token_values() {
414        let token = Token::new_on_new_line();
415        assert_eq!(token.start(), 0);
416        assert_eq!(token.end(), 0);
417        assert_eq!(token.kind(), Kind::Eof);
418        assert!(token.is_on_new_line());
419        assert!(!token.escaped());
420        assert!(!token.lone_surrogates());
421        assert!(!token.has_separator());
422    }
423
424    #[test]
425    fn token_creation_and_retrieval() {
426        let kind = Kind::Ident;
427        let start = 100u32;
428        let end = start + 5u32;
429        let is_on_new_line = true;
430        let escaped = false;
431        let lone_surrogates = true;
432        let has_separator = false;
433
434        let mut token = Token::default();
435        token.set_kind(kind);
436        token.set_start(start);
437        token.set_end(end);
438        token.set_is_on_new_line(is_on_new_line);
439        token.set_escaped(escaped);
440        token.set_lone_surrogates(lone_surrogates);
441        if has_separator {
442            // Assuming set_has_separator is not always called if false
443            token.set_has_separator(true);
444        }
445
446        assert_eq!(token.kind(), kind);
447        assert_eq!(token.start(), start);
448        assert_eq!(token.end(), end);
449        assert_eq!(token.is_on_new_line(), is_on_new_line);
450        assert_eq!(token.escaped(), escaped);
451        assert_eq!(token.lone_surrogates(), lone_surrogates);
452        assert_eq!(token.has_separator(), has_separator);
453    }
454
455    #[test]
456    fn token_setters() {
457        let mut token = Token::default();
458        token.set_kind(Kind::Ident);
459        token.set_span(Span::new(10, 15));
460        // is_on_new_line, escaped, lone_surrogates, has_separator are false by default from Token::default()
461
462        assert_eq!(token.start(), 10);
463        assert_eq!(token.end(), 15);
464        assert!(!token.escaped());
465        assert!(!token.is_on_new_line());
466        assert!(!token.lone_surrogates());
467
468        // Test set_end
469        let mut token_for_set_end = Token::default();
470        token_for_set_end.set_kind(Kind::Ident);
471        token_for_set_end.set_start(10);
472        token_for_set_end.set_end(15);
473
474        assert_eq!(token_for_set_end.end(), 15);
475        token_for_set_end.set_end(30);
476        assert_eq!(token_for_set_end.start(), 10);
477        assert_eq!(token_for_set_end.end(), 30);
478
479        // Test that other flags are not affected by set_start
480        let mut token_with_flags = Token::default();
481        token_with_flags.set_kind(Kind::Str);
482        token_with_flags.set_start(30);
483        token_with_flags.set_end(33);
484        token_with_flags.set_is_on_new_line(true);
485        token_with_flags.set_escaped(true);
486        token_with_flags.set_lone_surrogates(true);
487        token_with_flags.set_has_separator(true);
488
489        token_with_flags.set_start(40);
490        assert_eq!(token_with_flags.start(), 40);
491        assert!(token_with_flags.is_on_new_line());
492        assert!(token_with_flags.escaped());
493        assert!(token_with_flags.lone_surrogates());
494        assert!(token_with_flags.has_separator());
495
496        // Test that other flags are not affected by set_escaped
497        let mut token_with_flags2 = Token::default();
498        token_with_flags2.set_kind(Kind::Str);
499        token_with_flags2.set_start(50);
500        token_with_flags2.set_end(52);
501        token_with_flags2.set_is_on_new_line(true);
502        // escaped is false by default
503        token_with_flags2.set_lone_surrogates(true);
504        token_with_flags2.set_has_separator(true);
505
506        token_with_flags2.set_escaped(true);
507        assert_eq!(token_with_flags2.start(), 50);
508        assert!(token_with_flags2.is_on_new_line());
509        assert!(token_with_flags2.escaped());
510        assert!(token_with_flags2.lone_surrogates());
511        assert!(token_with_flags2.has_separator());
512        token_with_flags2.set_escaped(false);
513        assert!(!token_with_flags2.escaped());
514        assert!(token_with_flags2.is_on_new_line()); // Check again
515        assert!(token_with_flags2.lone_surrogates()); // Check again
516        assert!(token_with_flags2.has_separator()); // Check again
517
518        // Test set_is_on_new_line does not affect other flags
519        let mut token_flags_test_newline = Token::default();
520        token_flags_test_newline.set_kind(Kind::Str);
521        token_flags_test_newline.set_start(60);
522        token_flags_test_newline.set_end(62);
523        // is_on_new_line is false by default
524        token_flags_test_newline.set_escaped(true);
525        token_flags_test_newline.set_lone_surrogates(true);
526        token_flags_test_newline.set_has_separator(true);
527
528        token_flags_test_newline.set_is_on_new_line(true);
529        assert!(token_flags_test_newline.is_on_new_line());
530        assert_eq!(token_flags_test_newline.start(), 60);
531        assert!(token_flags_test_newline.escaped());
532        assert!(token_flags_test_newline.lone_surrogates());
533        assert!(token_flags_test_newline.has_separator());
534        token_flags_test_newline.set_is_on_new_line(false);
535        assert!(!token_flags_test_newline.is_on_new_line());
536        assert!(token_flags_test_newline.escaped());
537        assert!(token_flags_test_newline.lone_surrogates());
538        assert!(token_flags_test_newline.has_separator());
539
540        // Test set_lone_surrogates does not affect other flags
541        let mut token_flags_test_lone_surrogates = Token::default();
542        token_flags_test_lone_surrogates.set_kind(Kind::Str);
543        token_flags_test_lone_surrogates.set_start(70);
544        token_flags_test_lone_surrogates.set_end(72);
545        token_flags_test_lone_surrogates.set_is_on_new_line(true);
546        token_flags_test_lone_surrogates.set_escaped(true);
547        // lone_surrogates is false by default
548        token_flags_test_lone_surrogates.set_has_separator(true);
549
550        token_flags_test_lone_surrogates.set_lone_surrogates(true);
551        assert!(token_flags_test_lone_surrogates.lone_surrogates());
552        assert_eq!(token_flags_test_lone_surrogates.start(), 70);
553        assert!(token_flags_test_lone_surrogates.is_on_new_line());
554        assert!(token_flags_test_lone_surrogates.escaped());
555        assert!(token_flags_test_lone_surrogates.has_separator());
556        token_flags_test_lone_surrogates.set_lone_surrogates(false);
557        assert!(!token_flags_test_lone_surrogates.lone_surrogates());
558        assert!(token_flags_test_lone_surrogates.is_on_new_line());
559        assert!(token_flags_test_lone_surrogates.escaped());
560        assert!(token_flags_test_lone_surrogates.has_separator());
561    }
562
563    #[test]
564    fn is_on_new_line() {
565        let mut token = Token::default();
566        assert!(!token.is_on_new_line());
567        token.set_is_on_new_line(true);
568        assert!(token.is_on_new_line());
569        token.set_is_on_new_line(false);
570        assert!(!token.is_on_new_line());
571    }
572
573    #[test]
574    fn escaped() {
575        let mut token = Token::default();
576        assert!(!token.escaped());
577        token.set_escaped(true);
578        assert!(token.escaped());
579        token.set_escaped(false);
580        assert!(!token.escaped());
581    }
582
583    #[test]
584    fn lone_surrogates() {
585        let mut token = Token::default();
586        assert!(!token.lone_surrogates());
587        token.set_lone_surrogates(true);
588        assert!(token.lone_surrogates());
589        token.set_lone_surrogates(false);
590        assert!(!token.lone_surrogates());
591    }
592
593    #[test]
594    fn has_separator() {
595        let mut token = Token::default();
596        assert!(!token.has_separator());
597        token.set_has_separator(true);
598        assert!(token.has_separator());
599        token.set_has_separator(false);
600        assert!(!token.has_separator());
601    }
602}
oxc_parser/lexer/token.rs

oxc_parser/lexer/
token.rs
