use std::{fmt, mem, ptr::NonNull};
use oxc_span::Span;
use super::kind::Kind;
const START_SHIFT: usize = 0;
const END_SHIFT: usize = 32;
const KIND_SHIFT: usize = 64;
const IS_ON_NEW_LINE_SHIFT: usize = 72;
const ESCAPED_SHIFT: usize = 80;
const HAS_SEPARATOR_SHIFT: usize = 88;
const START_MASK: u128 = 0xFFFF_FFFF; const END_MASK: u128 = 0xFFFF_FFFF; const KIND_MASK: u128 = 0xFF; #[expect(dead_code)]
const BOOL_MASK: u128 = 0xFF;
const _: () = {
const fn is_valid_shift<T>(shift: usize) -> bool {
let align_bits = align_of::<T>() * 8;
shift.is_multiple_of(align_bits) && shift < u128::BITS as usize
}
assert!(is_valid_shift::<u32>(START_SHIFT));
assert!(is_valid_shift::<u32>(END_SHIFT));
assert!(size_of::<Kind>() == 1);
assert!(align_of::<Kind>() == 1);
assert!(is_valid_shift::<Kind>(KIND_SHIFT));
assert!(is_valid_shift::<bool>(IS_ON_NEW_LINE_SHIFT));
assert!(is_valid_shift::<bool>(ESCAPED_SHIFT));
assert!(is_valid_shift::<bool>(HAS_SEPARATOR_SHIFT));
};
#[derive(Clone, Copy, PartialEq, Eq)]
#[repr(transparent)]
pub struct Token(u128);
impl Default for Token {
#[inline]
fn default() -> Self {
const _: () = assert!(Kind::Eof as u8 == 0);
Self(0)
}
}
impl fmt::Debug for Token {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Token")
.field("kind", &self.kind())
.field("start", &self.start())
.field("end", &self.end())
.field("is_on_new_line", &self.is_on_new_line())
.field("escaped", &self.escaped())
.field("has_separator", &self.has_separator())
.finish()
}
}
impl Token {
#[inline]
pub(super) fn new(kind: Kind, start: u32, end: u32, pending: PendingToken) -> Self {
debug_assert!(end >= start);
Self(
(u128::from(start) << START_SHIFT)
| (u128::from(end) << END_SHIFT)
| ((kind as u128) << KIND_SHIFT)
| (u128::from(pending.0 & PendingToken::TOKEN_FLAGS) << IS_ON_NEW_LINE_SHIFT),
)
}
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub(super) struct PendingToken(u32);
impl PendingToken {
const IS_ON_NEW_LINE: u32 = 1;
const ESCAPED: u32 = 1 << (ESCAPED_SHIFT - IS_ON_NEW_LINE_SHIFT);
const HAS_SEPARATOR: u32 = 1 << (HAS_SEPARATOR_SHIFT - IS_ON_NEW_LINE_SHIFT);
const TOKEN_FLAGS: u32 = Self::IS_ON_NEW_LINE | Self::ESCAPED | Self::HAS_SEPARATOR;
const SAW_NEWLINE: u32 = 1 << 24;
const SAW_NEWLINE_FOR_COMMENT: u32 = 1 << 25;
const HAS_PENDING_COMMENTS: u32 = 1 << 26;
pub(super) const NONE: Self = Self(0);
pub(super) const START_OF_FILE: Self =
Self(Self::IS_ON_NEW_LINE | Self::SAW_NEWLINE | Self::SAW_NEWLINE_FOR_COMMENT);
#[inline]
pub(super) fn is_on_new_line(self) -> bool {
self.0 & Self::IS_ON_NEW_LINE != 0
}
#[inline]
pub(super) fn set_is_on_new_line(&mut self) {
self.0 |= Self::IS_ON_NEW_LINE;
}
#[inline]
pub(super) fn set_escaped(&mut self) {
self.0 |= Self::ESCAPED;
}
#[inline]
pub(super) fn set_has_separator(&mut self) {
self.0 |= Self::HAS_SEPARATOR;
}
#[inline]
pub(super) fn saw_newline(self) -> bool {
self.0 & Self::SAW_NEWLINE != 0
}
#[inline]
pub(super) fn saw_newline_for_comment(self) -> bool {
self.0 & Self::SAW_NEWLINE_FOR_COMMENT != 0
}
#[inline]
pub(super) fn set_saw_newline(&mut self) {
self.0 |= Self::SAW_NEWLINE | Self::SAW_NEWLINE_FOR_COMMENT;
}
#[inline]
pub(super) fn clear_saw_newline_for_comment(&mut self) {
self.0 &= !Self::SAW_NEWLINE_FOR_COMMENT;
}
#[inline]
pub(super) fn has_pending_comments(self) -> bool {
self.0 & Self::HAS_PENDING_COMMENTS != 0
}
#[inline]
pub(super) fn set_has_pending_comments(&mut self, value: bool) {
if value {
self.0 |= Self::HAS_PENDING_COMMENTS;
} else {
self.0 &= !Self::HAS_PENDING_COMMENTS;
}
}
}
impl Token {
#[inline]
pub fn span(&self) -> Span {
Span::new(self.start(), self.end())
}
#[cfg(feature = "mutate_tokens")]
#[inline]
pub fn set_span(&mut self, span: Span) {
self.set_span_impl(span);
}
#[cfg(not(feature = "mutate_tokens"))]
#[inline]
#[allow(dead_code, clippy::allow_attributes)]
pub(super) fn set_span(&mut self, span: Span) {
self.set_span_impl(span);
}
#[inline]
fn set_span_impl(&mut self, span: Span) {
self.set_start(span.start);
self.set_end(span.end);
}
#[inline]
pub fn start(&self) -> u32 {
((self.0 >> START_SHIFT) & START_MASK) as u32
}
#[inline]
pub(super) fn set_start(&mut self, start: u32) {
unsafe { self.write_u32(START_SHIFT, start) };
}
#[inline]
pub fn end(&self) -> u32 {
((self.0 >> END_SHIFT) & END_MASK) as u32
}
#[inline]
pub(super) fn set_end(&mut self, end: u32) {
let start = self.start();
debug_assert!(end >= start, "Token end ({end}) cannot be less than start ({start})");
unsafe { self.write_u32(END_SHIFT, end) };
}
#[inline]
pub fn kind(&self) -> Kind {
unsafe { mem::transmute::<u8, Kind>(((self.0 >> KIND_SHIFT) & KIND_MASK) as u8) }
}
#[cfg(feature = "mutate_tokens")]
#[inline]
pub fn set_kind(&mut self, kind: Kind) {
self.set_kind_impl(kind);
}
#[cfg(not(feature = "mutate_tokens"))]
#[inline]
pub(super) fn set_kind(&mut self, kind: Kind) {
self.set_kind_impl(kind);
}
#[inline]
fn set_kind_impl(&mut self, kind: Kind) {
const OFFSET: usize =
if cfg!(target_endian = "little") { KIND_SHIFT / 8 } else { 15 - (KIND_SHIFT / 8) };
unsafe { *NonNull::from(self).cast::<Kind>().add(OFFSET).as_mut() = kind };
}
#[inline]
pub fn is_on_new_line(&self) -> bool {
unsafe { self.read_bool(IS_ON_NEW_LINE_SHIFT) }
}
#[inline]
pub fn escaped(&self) -> bool {
unsafe { self.read_bool(ESCAPED_SHIFT) }
}
#[inline]
pub fn has_separator(&self) -> bool {
unsafe { self.read_bool(HAS_SEPARATOR_SHIFT) }
}
#[expect(clippy::inline_always)]
#[inline(always)] unsafe fn read_bool(&self, shift: usize) -> bool {
let offset = if cfg!(target_endian = "little") { shift / 8 } else { 15 - (shift / 8) };
unsafe {
let field_ptr = NonNull::from_ref(self).cast::<bool>().add(offset);
debug_assert!(field_ptr.cast::<u8>().read() <= 1);
*field_ptr.as_ref()
}
}
#[expect(clippy::inline_always)]
#[inline(always)] unsafe fn write_u32(&mut self, shift: usize, value: u32) {
let offset = if cfg!(target_endian = "little") { shift / 32 } else { 3 - (shift / 32) };
unsafe { *NonNull::from(self).cast::<u32>().add(offset).as_mut() = value };
}
}
#[cfg(test)]
mod test {
use super::{Kind, PendingToken, Span, Token};
const _: () = assert!(size_of::<Token>() == 16);
#[test]
fn default_token_values() {
let token = Token::default();
assert_eq!(token.start(), 0);
assert_eq!(token.end(), 0);
assert_eq!(token.kind(), Kind::Eof); assert!(!token.is_on_new_line());
assert!(!token.escaped());
assert!(!token.has_separator());
}
#[test]
fn new_token() {
let token = Token::new(Kind::Ident, 3, 7, PendingToken::NONE);
assert_eq!(token.kind(), Kind::Ident);
assert_eq!(token.span(), Span::new(3, 7));
assert!(!token.is_on_new_line());
assert!(!token.escaped());
assert!(!token.has_separator());
let token = Token::new(Kind::Eof, 0, 0, PendingToken::START_OF_FILE);
assert!(token.is_on_new_line());
assert!(!token.escaped());
assert!(!token.has_separator());
let mut pending = PendingToken::NONE;
pending.set_escaped();
pending.set_has_separator();
pending.set_has_pending_comments(true);
let mut token = Token::new(Kind::Str, 30, 33, pending);
assert!(!token.is_on_new_line());
assert!(token.escaped());
assert!(token.has_separator());
token.set_span(Span::new(10, 15));
token.set_kind(Kind::Ident);
assert_eq!(token.span(), Span::new(10, 15));
assert_eq!(token.kind(), Kind::Ident);
assert!(!token.is_on_new_line());
assert!(token.escaped());
assert!(token.has_separator());
}
#[test]
fn pending_token_trivia_state() {
let mut pending = PendingToken::NONE;
pending.set_saw_newline();
assert!(pending.saw_newline());
assert!(pending.saw_newline_for_comment());
pending.clear_saw_newline_for_comment();
assert!(pending.saw_newline());
assert!(!pending.saw_newline_for_comment());
pending.set_has_pending_comments(true);
assert!(pending.has_pending_comments());
pending.set_has_pending_comments(false);
assert!(!pending.has_pending_comments());
assert!(!pending.is_on_new_line());
}
}