use crate::clang::token::{SourceLoc, Token, TokenFlags, TokenKind};
use crate::clang::CLangStandard;
use std::fmt;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NumericRadix {
Binary = 2,
Octal = 8,
Decimal = 10,
Hex = 16,
}
impl NumericRadix {
#[inline]
pub fn base(self) -> u32 {
self as u32
}
pub fn prefix(self) -> Option<&'static str> {
match self {
Self::Binary => Some("0b"),
Self::Octal => None, Self::Decimal => None,
Self::Hex => Some("0x"),
}
}
pub fn is_valid_digit(self, ch: char) -> bool {
match self {
Self::Binary => ch == '0' || ch == '1',
Self::Octal => ('0'..='7').contains(&ch),
Self::Decimal => ch.is_ascii_digit(),
Self::Hex => ch.is_ascii_hexdigit(),
}
}
pub fn digit_value(self, ch: char) -> Option<u8> {
match self {
Self::Binary => {
if ch == '0' {
Some(0)
} else if ch == '1' {
Some(1)
} else {
None
}
}
Self::Octal => {
if ('0'..='7').contains(&ch) {
Some(ch as u8 - b'0')
} else {
None
}
}
Self::Decimal => {
if ch.is_ascii_digit() {
Some(ch as u8 - b'0')
} else {
None
}
}
Self::Hex => match ch {
'0'..='9' => Some(ch as u8 - b'0'),
'a'..='f' => Some(ch as u8 - b'a' + 10),
'A'..='F' => Some(ch as u8 - b'A' + 10),
_ => None,
},
}
}
}
impl fmt::Display for NumericRadix {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Binary => write!(f, "binary"),
Self::Octal => write!(f, "octal"),
Self::Decimal => write!(f, "decimal"),
Self::Hex => write!(f, "hexadecimal"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IntSuffixKind {
None,
Unsigned,
Long,
UnsignedLong,
LongLong,
UnsignedLongLong,
BitPrecise,
UnsignedBitPrecise,
Size,
UnsignedSize,
Int64,
Uint64,
Int128,
Uint128,
BitInt(u32),
UnsignedBitInt(u32),
Imaginary,
UnsignedImaginary,
LongImaginary,
}
impl IntSuffixKind {
pub fn is_unsigned(self) -> bool {
matches!(
self,
Self::Unsigned
| Self::UnsignedLong
| Self::UnsignedLongLong
| Self::UnsignedBitPrecise
| Self::UnsignedSize
| Self::Uint64
| Self::Uint128
| Self::UnsignedBitInt(_)
| Self::UnsignedImaginary
)
}
pub fn is_long(self) -> bool {
matches!(self, Self::Long | Self::UnsignedLong | Self::LongImaginary)
}
pub fn is_long_long(self) -> bool {
matches!(self, Self::LongLong | Self::UnsignedLongLong)
}
pub fn is_fixed_width(self) -> bool {
matches!(
self,
Self::Int64 | Self::Uint64 | Self::Int128 | Self::Uint128
)
}
pub fn is_bit_precise(self) -> bool {
matches!(
self,
Self::BitPrecise | Self::UnsignedBitPrecise | Self::BitInt(_) | Self::UnsignedBitInt(_)
)
}
pub fn is_c23(self) -> bool {
matches!(
self,
Self::BitPrecise
| Self::UnsignedBitPrecise
| Self::Size
| Self::UnsignedSize
| Self::BitInt(_)
| Self::UnsignedBitInt(_)
)
}
pub fn name(self) -> &'static str {
match self {
Self::None => "int",
Self::Unsigned => "unsigned int",
Self::Long => "long int",
Self::UnsignedLong => "unsigned long int",
Self::LongLong => "long long int",
Self::UnsignedLongLong => "unsigned long long int",
Self::BitPrecise => "_BitInt",
Self::UnsignedBitPrecise => "unsigned _BitInt",
Self::Size => "signed size_t",
Self::UnsignedSize => "size_t",
Self::Int64 => "__int64_t",
Self::Uint64 => "__uint64_t",
Self::Int128 => "__int128_t",
Self::Uint128 => "__uint128_t",
Self::BitInt(_) => "_BitInt(N)",
Self::UnsignedBitInt(_) => "unsigned _BitInt(N)",
Self::Imaginary => "_Imaginary int",
Self::UnsignedImaginary => "_Imaginary unsigned int",
Self::LongImaginary => "_Imaginary long int",
}
}
}
impl Default for IntSuffixKind {
fn default() -> Self {
Self::None
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FloatSuffixKind {
None,
Float,
LongDouble,
Float16,
Float32,
Float64,
Float128,
BFloat16,
ExplicitDouble,
Decimal32,
Decimal64,
Decimal128,
Float80,
Float128Quad,
Imaginary,
FloatImaginary,
LongDoubleImaginary,
}
impl FloatSuffixKind {
pub fn is_float(self) -> bool {
matches!(self, Self::Float | Self::Float16 | Self::Float32)
}
pub fn is_double(self) -> bool {
matches!(self, Self::None | Self::Float64 | Self::ExplicitDouble)
}
pub fn is_long_double(self) -> bool {
matches!(
self,
Self::LongDouble | Self::Float128 | Self::Float128Quad | Self::Float80
)
}
pub fn is_c23(self) -> bool {
matches!(
self,
Self::Float16 | Self::Float32 | Self::Float64 | Self::Float128 | Self::BFloat16
)
}
pub fn is_decimal(self) -> bool {
matches!(self, Self::Decimal32 | Self::Decimal64 | Self::Decimal128)
}
pub fn is_imaginary(self) -> bool {
matches!(
self,
Self::Imaginary | Self::FloatImaginary | Self::LongDoubleImaginary
)
}
pub fn name(self) -> &'static str {
match self {
Self::None => "double",
Self::Float => "float",
Self::LongDouble => "long double",
Self::Float16 => "_Float16",
Self::Float32 => "_Float32",
Self::Float64 => "_Float64",
Self::Float128 => "_Float128",
Self::BFloat16 => "__bf16",
Self::ExplicitDouble => "double",
Self::Decimal32 => "_Decimal32",
Self::Decimal64 => "_Decimal64",
Self::Decimal128 => "_Decimal128",
Self::Float80 => "__float80",
Self::Float128Quad => "__float128",
Self::Imaginary => "_Imaginary double",
Self::FloatImaginary => "_Imaginary float",
Self::LongDoubleImaginary => "_Imaginary long double",
}
}
}
impl Default for FloatSuffixKind {
fn default() -> Self {
Self::None
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NumericLiteralKind {
Integer,
Float,
HexFloat,
BinaryFloat,
Imaginary,
}
#[derive(Debug, Clone)]
pub struct NumericLiteralResult {
pub source: String,
pub significand: String,
pub radix: NumericRadix,
pub exponent: Option<i64>,
pub exponent_is_negative: bool,
pub int_suffix: IntSuffixKind,
pub float_suffix: FloatSuffixKind,
pub has_decimal_point: bool,
pub had_digit_separators: bool,
pub kind: NumericLiteralKind,
pub had_error: bool,
pub errors: Vec<String>,
}
impl NumericLiteralResult {
pub fn new(radix: NumericRadix) -> Self {
Self {
source: String::new(),
significand: String::new(),
radix,
exponent: None,
exponent_is_negative: false,
int_suffix: IntSuffixKind::None,
float_suffix: FloatSuffixKind::None,
has_decimal_point: false,
had_digit_separators: false,
kind: NumericLiteralKind::Integer,
had_error: false,
errors: Vec::new(),
}
}
pub fn is_floating(&self) -> bool {
matches!(
self.kind,
NumericLiteralKind::Float
| NumericLiteralKind::HexFloat
| NumericLiteralKind::BinaryFloat
| NumericLiteralKind::Imaginary
)
}
pub fn is_unsigned(&self) -> bool {
self.int_suffix.is_unsigned()
}
}
#[inline]
pub fn is_ident_start(ch: char) -> bool {
ch == '_' || ch.is_alphabetic()
}
#[inline]
pub fn is_ident_continue(ch: char) -> bool {
ch == '_' || ch.is_alphanumeric()
}
#[inline]
pub fn is_digit(ch: char) -> bool {
ch.is_ascii_digit()
}
#[inline]
pub fn is_hex_digit(ch: char) -> bool {
ch.is_ascii_hexdigit()
}
#[inline]
pub fn is_octal_digit(ch: char) -> bool {
('0'..='7').contains(&ch)
}
#[inline]
pub fn is_binary_digit(ch: char) -> bool {
ch == '0' || ch == '1'
}
#[inline]
pub fn hex_value(ch: char) -> u8 {
match ch {
'0'..='9' => ch as u8 - b'0',
'a'..='f' => ch as u8 - b'a' + 10,
'A'..='F' => ch as u8 - b'A' + 10,
_ => 0,
}
}
#[inline]
pub fn is_horizontal_whitespace(ch: char) -> bool {
ch == ' ' || ch == '\t'
}
#[inline]
pub fn is_whitespace(ch: char) -> bool {
ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\x0C'
}
pub struct NumericLiteralParser {
source: Vec<char>,
pos: usize,
standard: CLangStandard,
pub result: NumericLiteralResult,
}
impl NumericLiteralParser {
pub fn new(source: &str, standard: CLangStandard) -> Self {
let mut result = NumericLiteralResult::new(NumericRadix::Decimal);
result.source = source.to_string();
Self {
source: source.chars().collect(),
pos: 0,
standard,
result,
}
}
fn current(&self) -> Option<char> {
self.source.get(self.pos).copied()
}
fn peek(&self, n: usize) -> Option<char> {
self.source.get(self.pos + n).copied()
}
fn advance(&mut self) -> Option<char> {
let ch = self.current()?;
self.pos += 1;
Some(ch)
}
fn skip_if(&mut self, expected: char) -> bool {
if self.current() == Some(expected) {
self.advance();
true
} else {
false
}
}
fn skip_while(&mut self, predicate: fn(char) -> bool) {
while let Some(ch) = self.current() {
if !predicate(ch) {
break;
}
self.advance();
}
}
fn collect_while(&mut self, predicate: fn(char) -> bool) -> String {
let mut result = String::new();
while let Some(ch) = self.current() {
if !predicate(ch) {
break;
}
result.push(ch);
self.advance();
}
result
}
fn error(&mut self, msg: impl Into<String>) {
self.result.had_error = true;
self.result.errors.push(msg.into());
}
pub fn parse(&mut self) {
self.parse_radix_and_significand();
self.parse_exponent();
self.parse_suffix();
}
fn parse_radix_and_significand(&mut self) {
if self.current().is_none() {
self.error("empty numeric literal");
return;
}
let first = self.current().unwrap();
if first == '0' {
match self.peek(1) {
Some('x') | Some('X') => {
self.advance(); self.advance(); self.result.radix = NumericRadix::Hex;
self.parse_significand_digits();
}
Some('b') | Some('B') => {
self.advance(); self.advance(); self.result.radix = NumericRadix::Binary;
self.parse_significand_digits();
}
Some('o') | Some('O') => {
if self.standard == CLangStandard::C23 {
self.advance(); self.advance(); self.result.radix = NumericRadix::Octal;
self.parse_significand_digits();
} else {
self.result.radix = NumericRadix::Octal;
self.parse_significand_digits();
}
}
_ => {
self.advance(); self.result.significand.push('0');
if matches!(
self.current(),
Some('.') | Some('e') | Some('E') | Some('p') | Some('P')
) {
self.result.radix = NumericRadix::Decimal;
self.parse_decimal_fraction();
} else if self.current().is_some() && is_octal_digit(self.current().unwrap()) {
self.result.radix = NumericRadix::Octal;
self.parse_significand_digits();
} else {
self.result.radix = NumericRadix::Decimal;
}
}
}
} else if first == '.' && self.peek(1).map_or(false, is_digit) {
self.result.radix = NumericRadix::Decimal;
self.result.has_decimal_point = true;
self.advance(); self.parse_significand_digits();
} else {
self.result.radix = NumericRadix::Decimal;
self.parse_significand_digits();
}
}
fn parse_significand_digits(&mut self) {
let radix = self.result.radix;
let mut seen_any_digit = false;
loop {
match self.current() {
Some(ch) if radix.is_valid_digit(ch) => {
self.result.significand.push(ch);
seen_any_digit = true;
self.advance();
}
Some('\'') => {
let prev_is_digit = self
.result
.significand
.chars()
.last()
.map_or(false, |c| radix.is_valid_digit(c));
let next_is_digit = self.peek(1).map_or(false, |c| radix.is_valid_digit(c));
if prev_is_digit && next_is_digit {
self.result.had_digit_separators = true;
self.advance(); } else {
break;
}
}
Some('.') => {
if radix == NumericRadix::Decimal || radix == NumericRadix::Hex {
if self.result.has_decimal_point {
self.error("multiple decimal points in numeric literal");
self.advance();
} else {
self.result.has_decimal_point = true;
self.result.kind = NumericLiteralKind::Float;
self.advance();
}
} else {
break;
}
}
_ => break,
}
}
if !seen_any_digit && self.result.significand.is_empty() && !self.result.has_decimal_point {
self.error("numeric literal has no digits");
}
if self.result.has_decimal_point && self.result.significand.is_empty() {
}
}
fn parse_decimal_fraction(&mut self) {
self.result.radix = NumericRadix::Decimal;
if self.skip_if('.') {
self.result.has_decimal_point = true;
self.result.kind = NumericLiteralKind::Float;
self.parse_significand_digits();
}
}
fn parse_exponent(&mut self) {
match self.result.radix {
NumericRadix::Decimal => match self.current() {
Some('e') | Some('E') => {
self.result.kind = NumericLiteralKind::Float;
self.advance();
self.parse_exponent_value();
}
_ => {}
},
NumericRadix::Hex => match self.current() {
Some('p') | Some('P') => {
self.result.kind = NumericLiteralKind::HexFloat;
self.advance();
self.parse_exponent_value();
}
_ => {}
},
NumericRadix::Binary => match self.current() {
Some('p') | Some('P') => {
self.result.kind = NumericLiteralKind::BinaryFloat;
self.advance();
self.parse_exponent_value();
}
_ => {}
},
NumericRadix::Octal => {
match self.current() {
Some('p') | Some('P') => {
if self.result.has_decimal_point {
self.result.kind = NumericLiteralKind::Float;
self.advance();
self.parse_exponent_value();
}
}
_ => {}
}
}
}
}
fn parse_exponent_value(&mut self) {
if self.skip_if('+') {
self.result.exponent_is_negative = false;
} else if self.skip_if('-') {
self.result.exponent_is_negative = true;
}
let exp_str = self.collect_while(|ch| ch.is_ascii_digit());
if exp_str.is_empty() {
self.error("expected exponent digits");
return;
}
match exp_str.parse::<i64>() {
Ok(val) => {
self.result.exponent = Some(if self.result.exponent_is_negative {
-val
} else {
val
});
}
Err(_) => {
self.error("exponent value too large");
self.result.exponent = Some(0);
}
}
}
fn parse_suffix(&mut self) {
let mut is_imaginary = false;
match self.current() {
Some('i') | Some('I') | Some('j') | Some('J') => {
if matches!(self.current(), Some('i') | Some('I')) {
match self.peek(1) {
Some('6') if self.peek(2) == Some('4') => {
self.advance(); self.advance(); self.advance(); if self.result.is_floating() {
self.error("i64 suffix not valid on floating literal");
} else {
self.result.int_suffix = IntSuffixKind::Int64;
}
return;
}
Some('1') if self.peek(2) == Some('2') && self.peek(3) == Some('8') => {
self.advance(); self.advance(); self.advance(); self.advance(); if self.result.is_floating() {
self.error("i128 suffix not valid on floating literal");
} else {
self.result.int_suffix = IntSuffixKind::Int128;
}
return;
}
_ => {
if self.result.is_floating()
|| self.result.kind == NumericLiteralKind::Integer
{
is_imaginary = true;
self.advance();
if !self.result.is_floating() {
self.result.kind = NumericLiteralKind::Imaginary;
self.result.int_suffix = IntSuffixKind::Imaginary;
if self.result.int_suffix == IntSuffixKind::Unsigned {
self.result.int_suffix = IntSuffixKind::UnsignedImaginary;
} else if self.result.int_suffix == IntSuffixKind::Long {
self.result.int_suffix = IntSuffixKind::LongImaginary;
}
} else {
self.result.kind = NumericLiteralKind::Imaginary;
match self.result.float_suffix {
FloatSuffixKind::None => {
self.result.float_suffix = FloatSuffixKind::Imaginary;
}
FloatSuffixKind::Float => {
self.result.float_suffix =
FloatSuffixKind::FloatImaginary;
}
FloatSuffixKind::LongDouble => {
self.result.float_suffix =
FloatSuffixKind::LongDoubleImaginary;
}
_ => {}
}
}
return;
}
}
}
} else if matches!(self.current(), Some('j') | Some('J')) {
is_imaginary = true;
self.advance();
if !self.result.is_floating() {
self.result.kind = NumericLiteralKind::Imaginary;
self.result.int_suffix = IntSuffixKind::Imaginary;
} else {
self.result.kind = NumericLiteralKind::Imaginary;
match self.result.float_suffix {
FloatSuffixKind::None => {
self.result.float_suffix = FloatSuffixKind::Imaginary;
}
FloatSuffixKind::Float => {
self.result.float_suffix = FloatSuffixKind::FloatImaginary;
}
_ => {}
}
}
return;
}
}
_ => {}
}
if is_imaginary {
return;
}
let suffix_raw = self.collect_while(is_ident_continue);
if suffix_raw.is_empty() {
return;
}
if self.result.is_floating() {
self.parse_float_suffix(&suffix_raw);
} else {
self.parse_int_suffix(&suffix_raw);
}
}
fn parse_float_suffix(&mut self, suffix: &str) {
let suffix_lower: String = suffix.chars().map(|c| c.to_ascii_lowercase()).collect();
let s = suffix_lower.as_str();
match s {
"f" => self.result.float_suffix = FloatSuffixKind::Float,
"l" => self.result.float_suffix = FloatSuffixKind::LongDouble,
"f16" => self.result.float_suffix = FloatSuffixKind::Float16,
"f32" => self.result.float_suffix = FloatSuffixKind::Float32,
"f64" => self.result.float_suffix = FloatSuffixKind::Float64,
"f128" => self.result.float_suffix = FloatSuffixKind::Float128,
"bf16" => self.result.float_suffix = FloatSuffixKind::BFloat16,
"d" => self.result.float_suffix = FloatSuffixKind::ExplicitDouble,
"df" => self.result.float_suffix = FloatSuffixKind::Decimal32,
"dd" => self.result.float_suffix = FloatSuffixKind::Decimal64,
"dl" => self.result.float_suffix = FloatSuffixKind::Decimal128,
"w" => self.result.float_suffix = FloatSuffixKind::Float80,
"q" => self.result.float_suffix = FloatSuffixKind::Float128Quad,
"fi" => self.result.float_suffix = FloatSuffixKind::FloatImaginary,
"li" => self.result.float_suffix = FloatSuffixKind::LongDoubleImaginary,
_ => {
if suffix_lower.ends_with('i') {
let base = &suffix_lower[..suffix_lower.len() - 1];
match base {
"f" => self.result.float_suffix = FloatSuffixKind::FloatImaginary,
"l" => self.result.float_suffix = FloatSuffixKind::LongDoubleImaginary,
"" => self.result.float_suffix = FloatSuffixKind::Imaginary,
_ => {
self.error(format!("unknown floating suffix: {}", suffix));
}
}
} else {
self.error(format!("unknown floating suffix: {}", suffix));
}
}
}
}
fn parse_int_suffix(&mut self, suffix: &str) {
let suffix_lower: String = suffix.chars().map(|c| c.to_ascii_lowercase()).collect();
let s = suffix_lower.as_str();
if s.starts_with("_bitint") {
self.parse_bitint_suffix(suffix);
return;
}
let mut has_u = false;
let mut has_l = false;
let mut has_ll = false;
let mut has_z = false;
let mut has_wb = false;
let mut has_i64 = false;
let mut has_u64 = false;
let mut has_i128 = false;
let mut has_u128 = false;
for part in split_suffix_parts(s) {
match part {
"u" => has_u = true,
"l" => {
if has_l {
has_ll = true;
has_l = false;
} else {
has_l = true;
}
}
"ll" => has_ll = true,
"z" => has_z = true,
"wb" => has_wb = true,
"i64" => has_i64 = true,
"i128" => has_i128 = true,
"u64" => has_u64 = true,
"u128" => has_u128 = true,
_ => {
self.error(format!("unknown integer suffix component: {}", part));
return;
}
}
}
let component_count = [
has_u, has_l, has_ll, has_z, has_wb, has_i64, has_u64, has_i128, has_u128,
]
.iter()
.filter(|&&x| x)
.count() as u32;
if component_count > 2 {
self.error("too many components in integer suffix");
return;
}
match (
has_u, has_l, has_ll, has_z, has_wb, has_i64, has_u64, has_i128, has_u128,
) {
(false, false, false, false, false, false, false, false, false) => {
self.result.int_suffix = IntSuffixKind::None;
}
(true, false, false, false, false, false, false, false, false) => {
self.result.int_suffix = IntSuffixKind::Unsigned;
}
(false, true, false, false, false, false, false, false, false) => {
self.result.int_suffix = IntSuffixKind::Long;
}
(true, true, false, false, false, false, false, false, false) => {
self.result.int_suffix = IntSuffixKind::UnsignedLong;
}
(false, false, true, false, false, false, false, false, false) => {
self.result.int_suffix = IntSuffixKind::LongLong;
}
(true, false, true, false, false, false, false, false, false) => {
self.result.int_suffix = IntSuffixKind::UnsignedLongLong;
}
(false, false, false, true, false, false, false, false, false) => {
self.result.int_suffix = IntSuffixKind::Size;
}
(true, false, false, true, false, false, false, false, false) => {
self.result.int_suffix = IntSuffixKind::UnsignedSize;
}
(false, false, false, false, true, false, false, false, false) => {
self.result.int_suffix = IntSuffixKind::BitPrecise;
}
(true, false, false, false, true, false, false, false, false) => {
self.result.int_suffix = IntSuffixKind::UnsignedBitPrecise;
}
(false, false, false, false, false, true, false, false, false) => {
self.result.int_suffix = IntSuffixKind::Int64;
}
(false, false, false, false, false, false, true, false, false) => {
self.result.int_suffix = IntSuffixKind::Uint64;
}
(false, false, false, false, false, false, false, true, false) => {
self.result.int_suffix = IntSuffixKind::Int128;
}
(false, false, false, false, false, false, false, false, true) => {
self.result.int_suffix = IntSuffixKind::Uint128;
}
_ => {
self.error(format!("invalid integer suffix combination: {}", suffix));
}
}
}
fn parse_bitint_suffix(&mut self, suffix: &str) {
let rest = &suffix[7..];
if !rest.starts_with('(') {
self.result.int_suffix = IntSuffixKind::BitPrecise;
return;
}
let close_paren = rest.find(')');
match close_paren {
Some(idx) => {
let width_str = &rest[1..idx]; match width_str.parse::<u32>() {
Ok(width) => {
if width == 0 {
self.error("_BitInt width must be greater than zero");
} else if width > 0xFFFF {
self.error(format!(
"_BitInt width {} exceeds implementation limit",
width
));
}
self.result.int_suffix = IntSuffixKind::BitInt(width);
}
Err(_) => {
self.error(format!("invalid _BitInt width: {}", width_str));
}
}
}
None => {
self.error("unterminated _BitInt(N) suffix");
self.result.int_suffix = IntSuffixKind::BitPrecise;
}
}
}
}
fn split_suffix_parts(s: &str) -> Vec<&str> {
let mut parts = Vec::new();
let bytes = s.as_bytes();
let mut i = 0;
while i < bytes.len() {
match bytes[i] {
b'u' => {
parts.push("u");
i += 1;
}
b'l' => {
if i + 1 < bytes.len() && bytes[i + 1] == b'l' {
parts.push("ll");
i += 2;
} else {
parts.push("l");
i += 1;
}
}
b'z' => {
parts.push("z");
i += 1;
}
b'w' => {
if i + 1 < bytes.len() && bytes[i + 1] == b'b' {
parts.push("wb");
i += 2;
} else {
parts.push("w");
i += 1;
}
}
b'i' => {
if i + 2 < bytes.len() && bytes[i + 1] == b'6' && bytes[i + 2] == b'4' {
parts.push("i64");
i += 3;
} else if i + 3 < bytes.len()
&& bytes[i + 1] == b'1'
&& bytes[i + 2] == b'2'
&& bytes[i + 3] == b'8'
{
parts.push("i128");
i += 4;
} else {
parts.push("i");
i += 1;
}
}
_ => {
let start = i;
while i < bytes.len() && bytes[i].is_ascii_alphanumeric() {
i += 1;
}
parts.push(&s[start..i]);
}
}
}
parts
}
pub fn parse_numeric_literal(source: &str, standard: CLangStandard) -> NumericLiteralResult {
let mut parser = NumericLiteralParser::new(source, standard);
parser.parse();
parser.result
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CharPrefixKind {
Narrow,
Wide,
Utf8,
Utf16,
Utf32,
}
impl CharPrefixKind {
pub fn prefix_str(self) -> &'static str {
match self {
Self::Narrow => "",
Self::Wide => "L",
Self::Utf8 => "u8",
Self::Utf16 => "u",
Self::Utf32 => "U",
}
}
pub fn char_width(self) -> usize {
match self {
Self::Narrow => 1,
Self::Wide => 4,
Self::Utf8 => 1,
Self::Utf16 => 2,
Self::Utf32 => 4,
}
}
pub fn is_unicode(self) -> bool {
matches!(self, Self::Utf8 | Self::Utf16 | Self::Utf32)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum EscapeValue {
Literal(char),
Alert,
Backspace,
FormFeed,
Newline,
CarriageReturn,
HorizontalTab,
VerticalTab,
Backslash,
SingleQuote,
DoubleQuote,
QuestionMark,
Octal(u8),
Hex(u32),
UcnShort(u32),
UcnLong(u32),
Escape,
NamedEscape(String, u32),
}
impl EscapeValue {
pub fn codepoint(&self) -> u32 {
match *self {
Self::Literal(ch) => ch as u32,
Self::Alert => 0x07,
Self::Backspace => 0x08,
Self::FormFeed => 0x0C,
Self::Newline => 0x0A,
Self::CarriageReturn => 0x0D,
Self::HorizontalTab => 0x09,
Self::VerticalTab => 0x0B,
Self::Backslash => 0x5C,
Self::SingleQuote => 0x27,
Self::DoubleQuote => 0x22,
Self::QuestionMark => 0x3F,
Self::Octal(v) => v as u32,
Self::Hex(v) => v,
Self::UcnShort(v) => v,
Self::UcnLong(v) => v,
Self::Escape => 0x1B,
Self::NamedEscape(_, v) => v,
}
}
pub fn is_escaped(&self) -> bool {
!matches!(self, Self::Literal(_))
}
}
pub struct CharLiteralParser {
pub source: Vec<char>,
pos: usize,
pub prefix: CharPrefixKind,
pub value: u32,
pub multi_char_values: Vec<u8>,
pub is_multi_char: bool,
pub had_error: bool,
pub errors: Vec<String>,
pub is_complete: bool,
}
impl CharLiteralParser {
pub fn new(source: &str) -> Self {
Self {
source: source.chars().collect(),
pos: 0,
prefix: CharPrefixKind::Narrow,
value: 0,
multi_char_values: Vec::new(),
is_multi_char: false,
had_error: false,
errors: Vec::new(),
is_complete: false,
}
}
fn current(&self) -> Option<char> {
self.source.get(self.pos).copied()
}
fn peek(&self, n: usize) -> Option<char> {
self.source.get(self.pos + n).copied()
}
fn advance(&mut self) -> Option<char> {
let ch = self.current()?;
self.pos += 1;
Some(ch)
}
fn error(&mut self, msg: impl Into<String>) {
self.had_error = true;
self.errors.push(msg.into());
}
pub fn parse(&mut self) {
self.parse_prefix();
self.parse_body();
}
fn parse_prefix(&mut self) {
match self.current() {
Some('L') => {
self.prefix = CharPrefixKind::Wide;
self.advance();
}
Some('u') => {
if self.peek(1) == Some('8') {
self.prefix = CharPrefixKind::Utf8;
self.advance(); self.advance(); } else {
self.prefix = CharPrefixKind::Utf16;
self.advance(); }
}
Some('U') => {
self.prefix = CharPrefixKind::Utf32;
self.advance();
}
_ => {
self.prefix = CharPrefixKind::Narrow;
}
}
}
fn parse_body(&mut self) {
if self.current() != Some('\'') {
self.error("expected opening single quote");
return;
}
self.advance();
let mut chars_seen = 0u32;
loop {
match self.current() {
None => {
self.error("unterminated character literal");
return;
}
Some('\'') => {
self.is_complete = true;
self.advance();
break;
}
Some('\\') => {
self.advance(); if let Some(val) = self.parse_escape_sequence() {
let cp = val.codepoint();
if chars_seen == 0 {
self.value = cp;
}
chars_seen += 1;
if chars_seen > 1 {
self.is_multi_char = true;
self.value =
self.value.wrapping_shl(self.prefix.char_width() as u32 * 8)
| (cp & 0xFF);
}
if self.prefix == CharPrefixKind::Narrow {
if cp > 0xFF {
self.error(format!(
"character value 0x{:X} exceeds 8-bit char range",
cp
));
self.multi_char_values.push(cp as u8);
} else {
self.multi_char_values.push(cp as u8);
}
}
}
}
Some('\n') => {
self.error("newline in character literal");
self.advance();
}
Some(ch) => {
self.advance();
let cp = ch as u32;
if chars_seen == 0 {
self.value = cp;
}
chars_seen += 1;
if chars_seen > 1 {
self.is_multi_char = true;
self.value = self.value.wrapping_shl(self.prefix.char_width() as u32 * 8)
| (cp & 0xFF);
}
if self.prefix == CharPrefixKind::Narrow {
self.multi_char_values.push(cp as u8);
}
}
}
}
if chars_seen == 0 {
self.error("empty character literal");
}
}
fn parse_escape_sequence(&mut self) -> Option<EscapeValue> {
match self.current() {
None => {
self.error("incomplete escape sequence");
None
}
Some('a') => {
self.advance();
Some(EscapeValue::Alert)
}
Some('b') => {
self.advance();
Some(EscapeValue::Backspace)
}
Some('f') => {
self.advance();
Some(EscapeValue::FormFeed)
}
Some('n') => {
self.advance();
Some(EscapeValue::Newline)
}
Some('r') => {
self.advance();
Some(EscapeValue::CarriageReturn)
}
Some('t') => {
self.advance();
Some(EscapeValue::HorizontalTab)
}
Some('v') => {
self.advance();
Some(EscapeValue::VerticalTab)
}
Some('\\') => {
self.advance();
Some(EscapeValue::Backslash)
}
Some('\'') => {
self.advance();
Some(EscapeValue::SingleQuote)
}
Some('"') => {
self.advance();
Some(EscapeValue::DoubleQuote)
}
Some('?') => {
self.advance();
Some(EscapeValue::QuestionMark)
}
Some('e') => {
self.advance();
Some(EscapeValue::Escape) }
Some('0'..='7') => self.parse_octal_escape(),
Some('x') => self.parse_hex_escape(),
Some('u') => self.parse_ucn_short(),
Some('U') => self.parse_ucn_long(),
Some('N') => self.parse_named_escape(),
Some('\n') => {
self.advance();
None }
Some(ch) => {
self.advance();
Some(EscapeValue::Literal(ch))
}
}
}
fn parse_octal_escape(&mut self) -> Option<EscapeValue> {
let mut val: u16 = 0;
let mut count = 0;
while count < 3 {
match self.current() {
Some(ch) if ('0'..='7').contains(&ch) => {
val = val * 8 + (ch as u16 - '0' as u16);
count += 1;
self.advance();
}
_ => break,
}
}
if count == 0 {
self.error("expected octal digit after \\");
return None;
}
if val > 0xFF {
self.error(format!("octal escape value {} exceeds 0xFF", val));
Some(EscapeValue::Octal((val & 0xFF) as u8))
} else {
Some(EscapeValue::Octal(val as u8))
}
}
fn parse_hex_escape(&mut self) -> Option<EscapeValue> {
if self.current() == Some('x') {
self.advance();
}
let mut val: u32 = 0;
let mut count = 0;
loop {
match self.current() {
Some(ch) if ch.is_ascii_hexdigit() => {
val = val.wrapping_mul(16).wrapping_add(hex_value(ch) as u32);
count += 1;
self.advance();
let max_val = match self.prefix {
CharPrefixKind::Narrow | CharPrefixKind::Utf8 => 0xFFu32,
CharPrefixKind::Utf16 => 0xFFFFu32,
CharPrefixKind::Wide | CharPrefixKind::Utf32 => 0x10FFFFu32,
};
if val > max_val && count > 2 {
self.error(format!("hex escape 0x{:X} exceeds character range", val));
}
}
_ => break,
}
}
if count == 0 {
self.error("expected hex digit after \\x");
return None;
}
Some(EscapeValue::Hex(val))
}
fn parse_ucn_short(&mut self) -> Option<EscapeValue> {
if self.current() != Some('u') {
return None;
}
self.advance();
let mut val: u32 = 0;
let mut count = 0;
while count < 4 {
match self.current() {
Some(ch) if ch.is_ascii_hexdigit() => {
val = val * 16 + hex_value(ch) as u32;
count += 1;
self.advance();
}
_ => break,
}
}
if count != 4 {
self.error("\\u must be followed by exactly 4 hex digits");
return None;
}
if !is_valid_ucn_value(val, true) {
self.error(format!("invalid universal character name: U+{:04X}", val));
return None;
}
Some(EscapeValue::UcnShort(val))
}
fn parse_ucn_long(&mut self) -> Option<EscapeValue> {
if self.current() != Some('U') {
return None;
}
self.advance();
let mut val: u32 = 0;
let mut count = 0;
while count < 8 {
match self.current() {
Some(ch) if ch.is_ascii_hexdigit() => {
val = val * 16 + hex_value(ch) as u32;
count += 1;
self.advance();
}
_ => break,
}
}
if count != 8 {
self.error("\\U must be followed by exactly 8 hex digits");
return None;
}
if !is_valid_ucn_value(val, false) {
self.error(format!("invalid universal character name: U+{:08X}", val));
return None;
}
Some(EscapeValue::UcnLong(val))
}
fn parse_named_escape(&mut self) -> Option<EscapeValue> {
if self.current() != Some('N') {
return None;
}
self.advance();
if self.current() != Some('{') {
return Some(EscapeValue::Literal('N'));
}
self.advance();
let mut name = String::new();
let mut found_close = false;
while let Some(ch) = self.current() {
self.advance();
if ch == '}' {
found_close = true;
break;
}
if ch == '\n' {
self.error("newline in named escape sequence");
return None;
}
name.push(ch);
}
if !found_close {
self.error("unterminated named escape sequence (missing '}')");
return None;
}
if name.is_empty() {
self.error("empty named escape sequence");
return None;
}
let codepoint = lookup_named_character(&name);
match codepoint {
Some(cp) => Some(EscapeValue::NamedEscape(name, cp)),
None => {
self.error(format!("unknown character name: {}", name));
Some(EscapeValue::Literal('?'))
}
}
}
}
pub fn parse_char_literal(source: &str) -> CharLiteralParser {
let mut parser = CharLiteralParser::new(source);
parser.parse();
parser
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StringPrefixKind {
Narrow,
Wide,
Utf8,
Utf16,
Utf32,
}
impl StringPrefixKind {
pub fn prefix_str(self) -> &'static str {
match self {
Self::Narrow => "",
Self::Wide => "L",
Self::Utf8 => "u8",
Self::Utf16 => "u",
Self::Utf32 => "U",
}
}
pub fn char_width(self) -> usize {
match self {
Self::Narrow => 1,
Self::Wide => 4,
Self::Utf8 => 1, Self::Utf16 => 2,
Self::Utf32 => 4,
}
}
pub fn is_unicode(self) -> bool {
matches!(self, Self::Utf8 | Self::Utf16 | Self::Utf32)
}
}
pub struct StringLiteralParser {
pub source: Vec<char>,
pos: usize,
pub prefix: StringPrefixKind,
pub string_bytes: Vec<u8>,
pub string_codepoints: Vec<u32>,
pub is_raw: bool,
pub raw_delimiter: String,
pub num_concatenated: usize,
pub had_error: bool,
pub errors: Vec<String>,
pub is_complete: bool,
}
impl StringLiteralParser {
pub fn new(source: &str) -> Self {
Self {
source: source.chars().collect(),
pos: 0,
prefix: StringPrefixKind::Narrow,
string_bytes: Vec::new(),
string_codepoints: Vec::new(),
is_raw: false,
raw_delimiter: String::new(),
num_concatenated: 1,
had_error: false,
errors: Vec::new(),
is_complete: false,
}
}
fn current(&self) -> Option<char> {
self.source.get(self.pos).copied()
}
fn peek(&self, n: usize) -> Option<char> {
self.source.get(self.pos + n).copied()
}
fn advance(&mut self) -> Option<char> {
let ch = self.current()?;
self.pos += 1;
Some(ch)
}
fn error(&mut self, msg: impl Into<String>) {
self.had_error = true;
self.errors.push(msg.into());
}
pub fn parse(&mut self) {
self.parse_prefix();
if self.is_raw {
self.parse_raw_string_body();
} else {
self.parse_string_body();
}
}
fn parse_prefix(&mut self) {
match self.current() {
Some('L') => {
self.advance();
if self.current() == Some('R') {
self.advance(); self.is_raw = true;
}
self.prefix = StringPrefixKind::Wide;
}
Some('u') => {
self.advance();
if self.current() == Some('8') {
self.advance(); self.prefix = StringPrefixKind::Utf8;
if self.current() == Some('R') {
self.advance(); self.is_raw = true;
}
} else {
self.prefix = StringPrefixKind::Utf16;
if self.current() == Some('R') {
self.advance(); self.is_raw = true;
}
}
}
Some('U') => {
self.advance();
if self.current() == Some('R') {
self.advance(); self.is_raw = true;
}
self.prefix = StringPrefixKind::Utf32;
}
Some('R') => {
self.advance(); self.is_raw = true;
self.prefix = StringPrefixKind::Narrow;
}
_ => {
self.prefix = StringPrefixKind::Narrow;
}
}
}
fn parse_string_body(&mut self) {
if self.current() != Some('"') {
self.error("expected opening double quote in string literal");
return;
}
self.advance();
loop {
match self.current() {
None => {
self.error("unterminated string literal");
return;
}
Some('"') => {
self.is_complete = true;
self.advance();
break;
}
Some('\\') => {
self.advance(); if let Some(cp) = self.parse_string_escape() {
self.append_codepoint(cp);
}
}
Some('\n') => {
self.error("newline in string literal (use \\n for newline)");
self.advance();
}
Some(ch) => {
self.advance();
let cp = ch as u32;
self.append_codepoint(cp);
}
}
}
}
fn parse_string_escape(&mut self) -> Option<u32> {
match self.current() {
None => {
self.error("incomplete escape sequence in string");
None
}
Some('a') => {
self.advance();
Some(0x07)
}
Some('b') => {
self.advance();
Some(0x08)
}
Some('f') => {
self.advance();
Some(0x0C)
}
Some('n') => {
self.advance();
Some(0x0A)
}
Some('r') => {
self.advance();
Some(0x0D)
}
Some('t') => {
self.advance();
Some(0x09)
}
Some('v') => {
self.advance();
Some(0x0B)
}
Some('\\') => {
self.advance();
Some(0x5C)
}
Some('\'') => {
self.advance();
Some(0x27)
}
Some('"') => {
self.advance();
Some(0x22)
}
Some('?') => {
self.advance();
Some(0x3F)
}
Some('e') => {
self.advance();
Some(0x1B) }
Some('0'..='7') => self.parse_string_octal_escape(),
Some('x') => self.parse_string_hex_escape(),
Some('u') => self.parse_string_ucn_short(),
Some('U') => self.parse_string_ucn_long(),
Some('\n') => {
self.advance();
None }
Some(ch) => {
self.advance();
Some(ch as u32)
}
}
}
fn parse_string_octal_escape(&mut self) -> Option<u32> {
let mut val: u16 = 0;
let mut count = 0;
while count < 3 {
match self.current() {
Some(ch) if ('0'..='7').contains(&ch) => {
val = val * 8 + (ch as u16 - '0' as u16);
count += 1;
self.advance();
}
_ => break,
}
}
if count == 0 {
self.error("expected octal digit after \\");
return None;
}
Some(val as u32)
}
fn parse_string_hex_escape(&mut self) -> Option<u32> {
if self.current() == Some('x') {
self.advance();
}
let mut val: u32 = 0;
let mut count = 0;
loop {
match self.current() {
Some(ch) if ch.is_ascii_hexdigit() => {
val = val.wrapping_mul(16).wrapping_add(hex_value(ch) as u32);
count += 1;
self.advance();
}
_ => break,
}
}
if count == 0 {
self.error("expected hex digit after \\x");
return None;
}
Some(val)
}
fn parse_string_ucn_short(&mut self) -> Option<u32> {
if self.current() != Some('u') {
return None;
}
self.advance();
let mut val: u32 = 0;
let mut count = 0;
while count < 4 {
match self.current() {
Some(ch) if ch.is_ascii_hexdigit() => {
val = val * 16 + hex_value(ch) as u32;
count += 1;
self.advance();
}
_ => break,
}
}
if count != 4 {
self.error("\\u must be followed by exactly 4 hex digits");
return None;
}
if !is_valid_ucn_value(val, true) {
self.error(format!("invalid UCN: U+{:04X}", val));
return None;
}
Some(val)
}
fn parse_string_ucn_long(&mut self) -> Option<u32> {
if self.current() != Some('U') {
return None;
}
self.advance();
let mut val: u32 = 0;
let mut count = 0;
while count < 8 {
match self.current() {
Some(ch) if ch.is_ascii_hexdigit() => {
val = val * 16 + hex_value(ch) as u32;
count += 1;
self.advance();
}
_ => break,
}
}
if count != 8 {
self.error("\\U must be followed by exactly 8 hex digits");
return None;
}
if !is_valid_ucn_value(val, false) {
self.error(format!("invalid UCN: U+{:08X}", val));
return None;
}
Some(val)
}
fn parse_raw_string_body(&mut self) {
if self.current() != Some('"') {
self.error("expected '\"' to start raw string");
return;
}
self.advance();
let delimiter = self.collect_raw_delimiter();
if self.current() != Some('(') {
self.error("expected '(' after raw string delimiter");
return;
}
self.advance();
let closing_seq = format!("){}", delimiter);
let mut content = String::new();
loop {
match self.current() {
None => {
self.error("unterminated raw string literal");
return;
}
Some(')') => {
if self.match_str(&closing_seq) {
for _ in 0..closing_seq.len() {
self.advance();
}
if self.current() == Some('"') {
self.advance(); self.is_complete = true;
for ch in content.chars() {
let cp = ch as u32;
self.append_codepoint(cp);
if ch == '\n' {
}
}
return;
} else {
self.error("expected '\"' after raw string closing delimiter");
return;
}
} else {
let ch = self.advance().unwrap();
content.push(ch);
}
}
Some(ch) => {
content.push(ch);
self.advance();
}
}
}
}
fn collect_raw_delimiter(&mut self) -> String {
let mut delimiter = String::new();
while let Some(ch) = self.current() {
if ch == '(' {
break;
}
if ch == '"' || ch == '\\' || ch == ' ' || ch == '\t' || ch == '\n' {
self.error(format!(
"invalid character '{}' in raw string delimiter",
ch
));
break;
}
delimiter.push(ch);
self.advance();
}
self.raw_delimiter = delimiter.clone();
delimiter
}
fn match_str(&self, s: &str) -> bool {
let chars: Vec<char> = s.chars().collect();
for (i, &expected) in chars.iter().enumerate() {
if self.peek(i) != Some(expected) {
return false;
}
}
true
}
fn append_codepoint(&mut self, cp: u32) {
self.string_codepoints.push(cp);
match self.prefix {
StringPrefixKind::Narrow => {
if cp <= 0x7F {
self.string_bytes.push(cp as u8);
} else if cp <= 0xFF {
self.string_bytes.push(cp as u8);
} else {
self.string_bytes.push((cp & 0xFF) as u8);
if cp > 0xFF {
self.error(format!(
"character U+{:04X} doesn't fit in narrow string; truncated",
cp
));
}
}
}
StringPrefixKind::Utf8 => {
encode_utf8(cp, &mut self.string_bytes);
}
StringPrefixKind::Utf16 => {
if cp <= 0xFFFF {
self.string_bytes.push((cp & 0xFF) as u8);
self.string_bytes.push(((cp >> 8) & 0xFF) as u8);
} else if cp <= 0x10FFFF {
let high = 0xD800u32 + ((cp - 0x10000) >> 10);
let low = 0xDC00u32 + ((cp - 0x10000) & 0x3FF);
self.string_bytes.push((high & 0xFF) as u8);
self.string_bytes.push(((high >> 8) & 0xFF) as u8);
self.string_bytes.push((low & 0xFF) as u8);
self.string_bytes.push(((low >> 8) & 0xFF) as u8);
}
}
StringPrefixKind::Utf32 | StringPrefixKind::Wide => {
self.string_bytes.push((cp & 0xFF) as u8);
self.string_bytes.push(((cp >> 8) & 0xFF) as u8);
self.string_bytes.push(((cp >> 16) & 0xFF) as u8);
self.string_bytes.push(((cp >> 24) & 0xFF) as u8);
}
}
}
}
fn encode_utf8(cp: u32, buf: &mut Vec<u8>) {
if cp <= 0x7F {
buf.push(cp as u8);
} else if cp <= 0x7FF {
buf.push(0xC0 | ((cp >> 6) & 0x1F) as u8);
buf.push(0x80 | (cp & 0x3F) as u8);
} else if cp <= 0xFFFF {
buf.push(0xE0 | ((cp >> 12) & 0x0F) as u8);
buf.push(0x80 | ((cp >> 6) & 0x3F) as u8);
buf.push(0x80 | (cp & 0x3F) as u8);
} else if cp <= 0x10FFFF {
buf.push(0xF0 | ((cp >> 18) & 0x07) as u8);
buf.push(0x80 | ((cp >> 12) & 0x3F) as u8);
buf.push(0x80 | ((cp >> 6) & 0x3F) as u8);
buf.push(0x80 | (cp & 0x3F) as u8);
}
}
pub fn parse_string_literal(source: &str) -> StringLiteralParser {
let mut parser = StringLiteralParser::new(source);
parser.parse();
parser
}
pub fn parse_and_concatenate_strings(segments: &[&str]) -> StringLiteralParser {
if segments.is_empty() {
let mut parser = StringLiteralParser::new("");
parser.error("no string segments to concatenate");
return parser;
}
let mut combined = StringLiteralParser::new(segments[0]);
combined.parse();
combined.num_concatenated = segments.len() as usize;
for segment in &segments[1..] {
let mut next = StringLiteralParser::new(segment);
next.parse();
if next.prefix != combined.prefix {
if combined.prefix == StringPrefixKind::Narrow {
combined.prefix = next.prefix;
} else if next.prefix != StringPrefixKind::Narrow {
combined.error(format!(
"concatenating strings with incompatible prefixes: {:?} and {:?}",
combined.prefix, next.prefix
));
}
}
combined.string_bytes.extend(next.string_bytes);
combined.string_codepoints.extend(next.string_codepoints);
combined.is_complete = combined.is_complete && next.is_complete;
combined.had_error = combined.had_error || next.had_error;
combined.errors.extend(next.errors);
}
combined
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UcnValidity {
Valid,
ValidInLiteralOnly,
Invalid,
}
pub fn is_valid_ucn_for_identifier(codepoint: u32) -> bool {
if codepoint > 0x10FFFF {
return false;
}
if (0xD800..=0xDFFF).contains(&codepoint) {
return false;
}
if codepoint <= 0x7F {
return match codepoint {
0x24 | 0x40 | 0x60 => true,
_ => false,
};
}
if codepoint < 0xA0 {
return false;
}
if codepoint >= 0x7F && codepoint < 0xA0 {
return false;
}
is_in_c11_annex_d_allowed_range(codepoint)
}
pub fn validate_ucn(codepoint: u32, is_short_form: bool, in_identifier: bool) -> UcnValidity {
if codepoint > 0x10FFFF {
return UcnValidity::Invalid;
}
if (0xD800..=0xDFFF).contains(&codepoint) {
return UcnValidity::Invalid;
}
if is_short_form && codepoint > 0xFFFF {
return UcnValidity::Invalid;
}
if !in_identifier {
return UcnValidity::Valid;
}
if is_valid_ucn_for_identifier(codepoint) {
UcnValidity::Valid
} else if codepoint > 0xA0 || codepoint == 0x24 || codepoint == 0x40 || codepoint == 0x60 {
UcnValidity::ValidInLiteralOnly
} else {
UcnValidity::Invalid
}
}
pub fn is_valid_ucn_value(codepoint: u32, is_short_form: bool) -> bool {
if codepoint > 0x10FFFF {
return false;
}
if (0xD800..=0xDFFF).contains(&codepoint) {
return false;
}
if is_short_form && codepoint > 0xFFFF {
return false;
}
true
}
fn is_in_c11_annex_d_allowed_range(codepoint: u32) -> bool {
matches!(
codepoint,
0x00A8 | 0x00AA | 0x00AD | 0x00AF
| 0x00B2..=0x00B5
| 0x00B7..=0x00BA
| 0x00BC..=0x00BE
| 0x00C0..=0x00D6
| 0x00D8..=0x00F6
| 0x00F8..=0x00FF
) ||
(0x0100..=0x024F).contains(&codepoint) ||
(0x0250..=0x02FF).contains(&codepoint) ||
(0x0370..=0x03FF).contains(&codepoint) ||
(0x0400..=0x04FF).contains(&codepoint) ||
codepoint >= 0x00A0
}
pub struct UcnParser {
source: Vec<char>,
pos: usize,
pub codepoint: Option<u32>,
pub is_short_form: bool,
pub consumed: usize,
pub had_error: bool,
pub errors: Vec<String>,
}
impl UcnParser {
pub fn new(source: &str) -> Self {
Self {
source: source.chars().collect(),
pos: 0,
codepoint: None,
is_short_form: false,
consumed: 0,
had_error: false,
errors: Vec::new(),
}
}
fn current(&self) -> Option<char> {
self.source.get(self.pos).copied()
}
fn peek(&self, n: usize) -> Option<char> {
self.source.get(self.pos + n).copied()
}
fn advance(&mut self) -> Option<char> {
let ch = self.current()?;
self.pos += 1;
Some(ch)
}
fn error(&mut self, msg: impl Into<String>) {
self.had_error = true;
self.errors.push(msg.into());
}
pub fn parse(&mut self) -> bool {
let start_pos = self.pos;
if self.current() != Some('\\') {
self.error("UCN must start with backslash");
return false;
}
self.advance();
match self.current() {
Some('u') => {
self.is_short_form = true;
self.advance(); self.parse_hex_digits(4)
}
Some('U') => {
self.is_short_form = false;
self.advance(); self.parse_hex_digits(8)
}
_ => {
self.error("UCN must start with \\u or \\U");
false
}
};
self.consumed = self.pos - start_pos;
self.codepoint.is_some()
}
fn parse_hex_digits(&mut self, count: usize) -> bool {
let mut val: u32 = 0;
let mut digits_seen = 0;
while digits_seen < count {
match self.current() {
Some(ch) if ch.is_ascii_hexdigit() => {
val = val * 16 + hex_value(ch) as u32;
digits_seen += 1;
self.advance();
}
_ => {
self.error(format!(
"expected {} hex digits, got {}",
count, digits_seen
));
return false;
}
}
}
if !is_valid_ucn_value(val, self.is_short_form) {
self.error(format!(
"invalid UCN codepoint: U+{:0width$X}",
val,
width = if self.is_short_form { 4 } else { 8 }
));
self.codepoint = None;
return false;
}
self.codepoint = Some(val);
true
}
pub fn parse_for_identifier(&mut self) -> bool {
if !self.parse() {
return false;
}
if let Some(cp) = self.codepoint {
if !is_valid_ucn_for_identifier(cp) {
self.error(format!("UCN U+{:04X} is not valid in an identifier", cp));
return false;
}
}
true
}
}
pub fn parse_ucn(source: &str) -> UcnParser {
let mut parser = UcnParser::new(source);
parser.parse();
parser
}
pub fn parse_ucn_for_identifier(source: &str) -> UcnParser {
let mut parser = UcnParser::new(source);
parser.parse_for_identifier();
parser
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TrigraphEntry {
pub sequence: &'static str,
pub replacement: char,
pub name: &'static str,
}
pub static TRIGRAPH_TABLE: [TrigraphEntry; 9] = [
TrigraphEntry {
sequence: "??=",
replacement: '#',
name: "number sign",
},
TrigraphEntry {
sequence: "??(",
replacement: '[',
name: "left bracket",
},
TrigraphEntry {
sequence: "??/",
replacement: '\\',
name: "backslash",
},
TrigraphEntry {
sequence: "??)",
replacement: ']',
name: "right bracket",
},
TrigraphEntry {
sequence: "??'",
replacement: '^',
name: "circumflex",
},
TrigraphEntry {
sequence: "??<",
replacement: '{',
name: "left brace",
},
TrigraphEntry {
sequence: "??!",
replacement: '|',
name: "vertical bar",
},
TrigraphEntry {
sequence: "??>",
replacement: '}',
name: "right brace",
},
TrigraphEntry {
sequence: "??-",
replacement: '~',
name: "tilde",
},
];
pub fn is_trigraph(s: &str) -> Option<(char, usize)> {
if s.len() < 3 {
return None;
}
let bytes = s.as_bytes();
if bytes[0] != b'?' || bytes[1] != b'?' {
return None;
}
for entry in &TRIGRAPH_TABLE {
if s.starts_with(entry.sequence) {
return Some((entry.replacement, 3));
}
}
None
}
pub fn replace_trigraphs(source: &str) -> String {
let mut result = String::with_capacity(source.len());
let chars: Vec<char> = source.chars().collect();
let mut i = 0;
while i < chars.len() {
if i + 2 < chars.len() && chars[i] == '?' && chars[i + 1] == '?' {
let maybe_trigraph: String = chars[i..i + 3].iter().collect();
let mut found = false;
for entry in &TRIGRAPH_TABLE {
if maybe_trigraph == entry.sequence {
result.push(entry.replacement);
i += 3;
found = true;
break;
}
}
if !found {
result.push(chars[i]);
i += 1;
}
} else {
result.push(chars[i]);
i += 1;
}
}
result
}
pub fn count_trigraphs(source: &str) -> usize {
let mut count = 0;
let chars: Vec<char> = source.chars().collect();
let mut i = 0;
while i + 2 < chars.len() {
if chars[i] == '?' && chars[i + 1] == '?' {
let maybe_trigraph: String = chars[i..i + 3].iter().collect();
for entry in &TRIGRAPH_TABLE {
if maybe_trigraph == entry.sequence {
count += 1;
i += 2; break;
}
}
}
i += 1;
}
count
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DigraphEntry {
pub sequence: &'static str,
pub token_kind: TokenKind,
pub equivalent: &'static str,
pub length: usize,
}
pub static DIGRAPH_TABLE: [DigraphEntry; 6] = [
DigraphEntry {
sequence: "<:",
token_kind: TokenKind::LBracket,
equivalent: "[",
length: 2,
},
DigraphEntry {
sequence: ":>",
token_kind: TokenKind::RBracket,
equivalent: "]",
length: 2,
},
DigraphEntry {
sequence: "<%",
token_kind: TokenKind::LBrace,
equivalent: "{",
length: 2,
},
DigraphEntry {
sequence: "%>",
token_kind: TokenKind::RBrace,
equivalent: "}",
length: 2,
},
DigraphEntry {
sequence: "%:",
token_kind: TokenKind::Hash,
equivalent: "#",
length: 2,
},
DigraphEntry {
sequence: "%:%:",
token_kind: TokenKind::HashHash,
equivalent: "##",
length: 4,
},
];
pub fn is_digraph(s: &str) -> Option<(TokenKind, usize)> {
for entry in &DIGRAPH_TABLE {
if s.starts_with(entry.sequence) {
return Some((entry.token_kind, entry.length));
}
}
None
}
pub fn is_digraph_hashhash(s: &str) -> bool {
s.starts_with("%:%:")
}
pub fn lookup_digraph(sequence: &str) -> Option<&'static DigraphEntry> {
DIGRAPH_TABLE
.iter()
.find(|entry| entry.sequence == sequence)
}
pub fn count_digraphs(source: &str) -> usize {
let mut count = 0;
let chars: Vec<char> = source.chars().collect();
let mut i = 0;
while i < chars.len() {
for entry in &DIGRAPH_TABLE {
let seq_chars: Vec<char> = entry.sequence.chars().collect();
if i + seq_chars.len() <= chars.len() {
let slice: String = chars[i..i + seq_chars.len()].iter().collect();
if slice == entry.sequence {
count += 1;
i += seq_chars.len() - 1; break;
}
}
}
i += 1;
}
count
}
#[derive(Debug, Clone)]
pub struct IntSuffixEntry {
pub suffix: &'static str,
pub spellings: &'static [&'static str],
pub kind: IntSuffixKind,
pub introduced_in: &'static str,
pub description: &'static str,
}
pub static INT_SUFFIX_TABLE: &[IntSuffixEntry] = &[
IntSuffixEntry {
suffix: "",
spellings: &[""],
kind: IntSuffixKind::None,
introduced_in: "C89",
description: "int (signed int)",
},
IntSuffixEntry {
suffix: "l",
spellings: &["l", "L"],
kind: IntSuffixKind::Long,
introduced_in: "C89",
description: "long int",
},
IntSuffixEntry {
suffix: "lu",
spellings: &["lu", "LU", "Lu", "lU", "ul", "UL", "Ul", "uL"],
kind: IntSuffixKind::UnsignedLong,
introduced_in: "C89",
description: "unsigned long int",
},
IntSuffixEntry {
suffix: "u",
spellings: &["u", "U"],
kind: IntSuffixKind::Unsigned,
introduced_in: "C89",
description: "unsigned int",
},
IntSuffixEntry {
suffix: "ll",
spellings: &["ll", "LL", "Ll", "lL"],
kind: IntSuffixKind::LongLong,
introduced_in: "C99",
description: "long long int",
},
IntSuffixEntry {
suffix: "llu",
spellings: &["llu", "LLU", "ull", "ULL", "LLu", "uLL"],
kind: IntSuffixKind::UnsignedLongLong,
introduced_in: "C99",
description: "unsigned long long int",
},
IntSuffixEntry {
suffix: "i",
spellings: &["i", "I"],
kind: IntSuffixKind::Imaginary,
introduced_in: "GNU",
description: "_Imaginary int",
},
IntSuffixEntry {
suffix: "i128",
spellings: &["i128", "I128"],
kind: IntSuffixKind::Int128,
introduced_in: "GNU",
description: "__int128_t (128-bit signed integer)",
},
IntSuffixEntry {
suffix: "i64",
spellings: &["i64", "I64"],
kind: IntSuffixKind::Int64,
introduced_in: "GNU",
description: "__int64_t (64-bit signed integer)",
},
IntSuffixEntry {
suffix: "il",
spellings: &["il", "IL", "iL", "Il"],
kind: IntSuffixKind::LongImaginary,
introduced_in: "GNU",
description: "_Imaginary long int",
},
IntSuffixEntry {
suffix: "iu",
spellings: &["iu", "IU", "iU", "Iu", "ui", "UI", "Ui", "uI"],
kind: IntSuffixKind::UnsignedImaginary,
introduced_in: "GNU",
description: "_Imaginary unsigned int",
},
IntSuffixEntry {
suffix: "u128",
spellings: &["u128", "U128"],
kind: IntSuffixKind::Uint128,
introduced_in: "GNU",
description: "__uint128_t (128-bit unsigned integer)",
},
IntSuffixEntry {
suffix: "u64",
spellings: &["u64", "U64"],
kind: IntSuffixKind::Uint64,
introduced_in: "GNU",
description: "__uint64_t (64-bit unsigned integer)",
},
IntSuffixEntry {
suffix: "uz",
spellings: &["uz", "UZ", "zu", "ZU", "Uz", "Z", "z"],
kind: IntSuffixKind::UnsignedSize,
introduced_in: "C23",
description: "size_t-width unsigned integer",
},
IntSuffixEntry {
suffix: "uwb",
spellings: &["uwb", "UWB", "Uwb", "uWB"],
kind: IntSuffixKind::UnsignedBitPrecise,
introduced_in: "C23",
description: "unsigned _BitInt (bit-precise, width from value)",
},
IntSuffixEntry {
suffix: "wb",
spellings: &["wb", "WB"],
kind: IntSuffixKind::BitPrecise,
introduced_in: "C23",
description: "_BitInt (bit-precise, width from value)",
},
IntSuffixEntry {
suffix: "z",
spellings: &["z", "Z"],
kind: IntSuffixKind::Size,
introduced_in: "C23",
description: "size_t-width signed integer",
},
];
#[derive(Debug, Clone)]
pub struct FloatSuffixEntry {
pub suffix: &'static str,
pub spellings: &'static [&'static str],
pub kind: FloatSuffixKind,
pub introduced_in: &'static str,
pub description: &'static str,
pub ieee_format: &'static str,
}
pub static FLOAT_SUFFIX_TABLE: &[FloatSuffixEntry] = &[
FloatSuffixEntry {
suffix: "",
spellings: &[""],
kind: FloatSuffixKind::None,
introduced_in: "C89",
description: "double",
ieee_format: "binary64",
},
FloatSuffixEntry {
suffix: "f",
spellings: &["f", "F"],
kind: FloatSuffixKind::Float,
introduced_in: "C89",
description: "float",
ieee_format: "binary32",
},
FloatSuffixEntry {
suffix: "l",
spellings: &["l", "L"],
kind: FloatSuffixKind::LongDouble,
introduced_in: "C89",
description: "long double (80-bit x87 or 128-bit IEEE)",
ieee_format: "binary80 or binary128",
},
FloatSuffixEntry {
suffix: "f16",
spellings: &["f16", "F16"],
kind: FloatSuffixKind::Float16,
introduced_in: "C23",
description: "_Float16 (IEEE binary16)",
ieee_format: "binary16",
},
FloatSuffixEntry {
suffix: "f32",
spellings: &["f32", "F32"],
kind: FloatSuffixKind::Float32,
introduced_in: "C23",
description: "_Float32 (IEEE binary32, same as float)",
ieee_format: "binary32",
},
FloatSuffixEntry {
suffix: "f64",
spellings: &["f64", "F64"],
kind: FloatSuffixKind::Float64,
introduced_in: "C23",
description: "_Float64 (IEEE binary64, same as double)",
ieee_format: "binary64",
},
FloatSuffixEntry {
suffix: "f128",
spellings: &["f128", "F128"],
kind: FloatSuffixKind::Float128,
introduced_in: "C23",
description: "_Float128 (IEEE binary128)",
ieee_format: "binary128",
},
FloatSuffixEntry {
suffix: "bf16",
spellings: &["bf16", "BF16"],
kind: FloatSuffixKind::BFloat16,
introduced_in: "C23",
description: "__bf16 (brain floating-point 16-bit)",
ieee_format: "bfloat16",
},
FloatSuffixEntry {
suffix: "d",
spellings: &["d", "D"],
kind: FloatSuffixKind::ExplicitDouble,
introduced_in: "C++",
description: "double (explicit)",
ieee_format: "binary64",
},
FloatSuffixEntry {
suffix: "df",
spellings: &["df", "DF"],
kind: FloatSuffixKind::Decimal32,
introduced_in: "GNU/TR24732",
description: "_Decimal32 (decimal 32-bit float)",
ieee_format: "decimal32",
},
FloatSuffixEntry {
suffix: "dd",
spellings: &["dd", "DD"],
kind: FloatSuffixKind::Decimal64,
introduced_in: "GNU/TR24732",
description: "_Decimal64 (decimal 64-bit float)",
ieee_format: "decimal64",
},
FloatSuffixEntry {
suffix: "dl",
spellings: &["dl", "DL"],
kind: FloatSuffixKind::Decimal128,
introduced_in: "GNU/TR24732",
description: "_Decimal128 (decimal 128-bit float)",
ieee_format: "decimal128",
},
FloatSuffixEntry {
suffix: "w",
spellings: &["w", "W"],
kind: FloatSuffixKind::Float80,
introduced_in: "GCC",
description: "__float80 (x86 80-bit extended precision)",
ieee_format: "binary80",
},
FloatSuffixEntry {
suffix: "q",
spellings: &["q", "Q"],
kind: FloatSuffixKind::Float128Quad,
introduced_in: "GCC",
description: "__float128 (128-bit quad precision)",
ieee_format: "binary128",
},
FloatSuffixEntry {
suffix: "fi",
spellings: &["fi", "FI", "if", "IF"],
kind: FloatSuffixKind::FloatImaginary,
introduced_in: "GNU",
description: "_Imaginary float",
ieee_format: "binary32 imaginary",
},
FloatSuffixEntry {
suffix: "i",
spellings: &["i", "I", "j", "J"],
kind: FloatSuffixKind::Imaginary,
introduced_in: "GNU",
description: "_Imaginary double",
ieee_format: "binary64 imaginary",
},
FloatSuffixEntry {
suffix: "li",
spellings: &["li", "LI", "il", "IL"],
kind: FloatSuffixKind::LongDoubleImaginary,
introduced_in: "GNU",
description: "_Imaginary long double",
ieee_format: "binary80 imaginary",
},
];
pub fn lookup_int_suffix(suffix: &str) -> Option<&'static IntSuffixEntry> {
let suffix_lower: String = suffix.chars().map(|c| c.to_ascii_lowercase()).collect();
INT_SUFFIX_TABLE
.iter()
.find(|entry| entry.spellings.contains(&suffix_lower.as_str()))
}
pub fn lookup_float_suffix(suffix: &str) -> Option<&'static FloatSuffixEntry> {
let suffix_lower: String = suffix.chars().map(|c| c.to_ascii_lowercase()).collect();
FLOAT_SUFFIX_TABLE
.iter()
.find(|entry| entry.spellings.contains(&suffix_lower.as_str()))
}
pub fn all_int_suffix_spellings() -> Vec<&'static str> {
INT_SUFFIX_TABLE
.iter()
.flat_map(|entry| entry.spellings.iter().copied())
.collect()
}
pub fn all_float_suffix_spellings() -> Vec<&'static str> {
FLOAT_SUFFIX_TABLE
.iter()
.flat_map(|entry| entry.spellings.iter().copied())
.collect()
}
#[derive(Debug, Clone)]
pub struct OperatorDescriptor {
pub spelling: &'static str,
pub kind: TokenKind,
pub length: usize,
pub is_preprocessor: bool,
pub is_assignment: bool,
pub is_comparison: bool,
pub precedence: u8,
pub left_associative: bool,
}
pub static OPERATOR_TABLE: &[OperatorDescriptor] = &[
OperatorDescriptor {
spelling: "<<=",
kind: TokenKind::LessLessEqual,
length: 3,
is_preprocessor: false,
is_assignment: true,
is_comparison: false,
precedence: 2,
left_associative: true,
},
OperatorDescriptor {
spelling: ">>=",
kind: TokenKind::GreaterGreaterEqual,
length: 3,
is_preprocessor: false,
is_assignment: true,
is_comparison: false,
precedence: 2,
left_associative: true,
},
OperatorDescriptor {
spelling: "<=>",
kind: TokenKind::Spaceship,
length: 3,
is_preprocessor: false,
is_assignment: false,
is_comparison: true,
precedence: 8,
left_associative: true,
},
OperatorDescriptor {
spelling: "->",
kind: TokenKind::Arrow,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: true,
},
OperatorDescriptor {
spelling: "++",
kind: TokenKind::PlusPlus,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: true,
},
OperatorDescriptor {
spelling: "--",
kind: TokenKind::MinusMinus,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: true,
},
OperatorDescriptor {
spelling: "<<",
kind: TokenKind::LessLess,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 5,
left_associative: true,
},
OperatorDescriptor {
spelling: ">>",
kind: TokenKind::GreaterGreater,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 5,
left_associative: true,
},
OperatorDescriptor {
spelling: "<=",
kind: TokenKind::LessEqual,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: true,
precedence: 8,
left_associative: true,
},
OperatorDescriptor {
spelling: ">=",
kind: TokenKind::GreaterEqual,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: true,
precedence: 8,
left_associative: true,
},
OperatorDescriptor {
spelling: "==",
kind: TokenKind::EqualEqual,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: true,
precedence: 9,
left_associative: true,
},
OperatorDescriptor {
spelling: "!=",
kind: TokenKind::NotEqual,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: true,
precedence: 9,
left_associative: true,
},
OperatorDescriptor {
spelling: "&&",
kind: TokenKind::AndAnd,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 13,
left_associative: true,
},
OperatorDescriptor {
spelling: "||",
kind: TokenKind::OrOr,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 14,
left_associative: true,
},
OperatorDescriptor {
spelling: "+=",
kind: TokenKind::PlusEqual,
length: 2,
is_preprocessor: false,
is_assignment: true,
is_comparison: false,
precedence: 2,
left_associative: true,
},
OperatorDescriptor {
spelling: "-=",
kind: TokenKind::MinusEqual,
length: 2,
is_preprocessor: false,
is_assignment: true,
is_comparison: false,
precedence: 2,
left_associative: true,
},
OperatorDescriptor {
spelling: "*=",
kind: TokenKind::StarEqual,
length: 2,
is_preprocessor: false,
is_assignment: true,
is_comparison: false,
precedence: 2,
left_associative: true,
},
OperatorDescriptor {
spelling: "/=",
kind: TokenKind::SlashEqual,
length: 2,
is_preprocessor: false,
is_assignment: true,
is_comparison: false,
precedence: 2,
left_associative: true,
},
OperatorDescriptor {
spelling: "%=",
kind: TokenKind::PercentEqual,
length: 2,
is_preprocessor: false,
is_assignment: true,
is_comparison: false,
precedence: 2,
left_associative: true,
},
OperatorDescriptor {
spelling: "&=",
kind: TokenKind::AmpersandEqual,
length: 2,
is_preprocessor: false,
is_assignment: true,
is_comparison: false,
precedence: 2,
left_associative: true,
},
OperatorDescriptor {
spelling: "|=",
kind: TokenKind::PipeEqual,
length: 2,
is_preprocessor: false,
is_assignment: true,
is_comparison: false,
precedence: 2,
left_associative: true,
},
OperatorDescriptor {
spelling: "^=",
kind: TokenKind::CaretEqual,
length: 2,
is_preprocessor: false,
is_assignment: true,
is_comparison: false,
precedence: 2,
left_associative: true,
},
OperatorDescriptor {
spelling: "::",
kind: TokenKind::ScopeResolution,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: true,
},
OperatorDescriptor {
spelling: ".*",
kind: TokenKind::DotStar,
length: 2,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: true,
},
OperatorDescriptor {
spelling: "->*",
kind: TokenKind::ArrowStar,
length: 3,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: true,
},
OperatorDescriptor {
spelling: "...",
kind: TokenKind::Ellipsis,
length: 3,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 0,
left_associative: false,
},
OperatorDescriptor {
spelling: "##",
kind: TokenKind::HashHash,
length: 2,
is_preprocessor: true,
is_assignment: false,
is_comparison: false,
precedence: 0,
left_associative: false,
},
OperatorDescriptor {
spelling: "#",
kind: TokenKind::Hash,
length: 1,
is_preprocessor: true,
is_assignment: false,
is_comparison: false,
precedence: 0,
left_associative: false,
},
OperatorDescriptor {
spelling: "+",
kind: TokenKind::Plus,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 4,
left_associative: true,
},
OperatorDescriptor {
spelling: "-",
kind: TokenKind::Minus,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 4,
left_associative: true,
},
OperatorDescriptor {
spelling: "*",
kind: TokenKind::Star,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 3,
left_associative: true,
},
OperatorDescriptor {
spelling: "/",
kind: TokenKind::Slash,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 3,
left_associative: true,
},
OperatorDescriptor {
spelling: "%",
kind: TokenKind::Percent,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 3,
left_associative: true,
},
OperatorDescriptor {
spelling: "&",
kind: TokenKind::Ampersand,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 6,
left_associative: true,
},
OperatorDescriptor {
spelling: "|",
kind: TokenKind::Pipe,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 10,
left_associative: true,
},
OperatorDescriptor {
spelling: "^",
kind: TokenKind::Caret,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 11,
left_associative: true,
},
OperatorDescriptor {
spelling: "~",
kind: TokenKind::Tilde,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: false,
},
OperatorDescriptor {
spelling: "!",
kind: TokenKind::Exclaim,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: false,
},
OperatorDescriptor {
spelling: "<",
kind: TokenKind::Less,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: true,
precedence: 8,
left_associative: true,
},
OperatorDescriptor {
spelling: ">",
kind: TokenKind::Greater,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: true,
precedence: 8,
left_associative: true,
},
OperatorDescriptor {
spelling: "=",
kind: TokenKind::Equal,
length: 1,
is_preprocessor: false,
is_assignment: true,
is_comparison: false,
precedence: 2,
left_associative: true,
},
OperatorDescriptor {
spelling: "(",
kind: TokenKind::LParen,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: true,
},
OperatorDescriptor {
spelling: ")",
kind: TokenKind::RParen,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: true,
},
OperatorDescriptor {
spelling: "{",
kind: TokenKind::LBrace,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 0,
left_associative: false,
},
OperatorDescriptor {
spelling: "}",
kind: TokenKind::RBrace,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 0,
left_associative: false,
},
OperatorDescriptor {
spelling: "[",
kind: TokenKind::LBracket,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: true,
},
OperatorDescriptor {
spelling: "]",
kind: TokenKind::RBracket,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: true,
},
OperatorDescriptor {
spelling: ".",
kind: TokenKind::Dot,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 1,
left_associative: true,
},
OperatorDescriptor {
spelling: ";",
kind: TokenKind::Semicolon,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 0,
left_associative: false,
},
OperatorDescriptor {
spelling: ",",
kind: TokenKind::Comma,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 15,
left_associative: true,
},
OperatorDescriptor {
spelling: ":",
kind: TokenKind::Colon,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 0,
left_associative: false,
},
OperatorDescriptor {
spelling: "?",
kind: TokenKind::Question,
length: 1,
is_preprocessor: false,
is_assignment: false,
is_comparison: false,
precedence: 14,
left_associative: false,
},
];
pub fn try_lex_operator(source: &str) -> Option<(TokenKind, usize)> {
for entry in OPERATOR_TABLE.iter().filter(|e| e.length == 3) {
if source.starts_with(entry.spelling) {
return Some((entry.kind, entry.length));
}
}
for entry in OPERATOR_TABLE.iter().filter(|e| e.length == 2) {
if source.starts_with(entry.spelling) {
return Some((entry.kind, entry.length));
}
}
for entry in OPERATOR_TABLE.iter().filter(|e| e.length == 1) {
if source.starts_with(entry.spelling) {
return Some((entry.kind, entry.length));
}
}
None
}
pub fn lookup_operator_by_spelling(spelling: &str) -> Option<&'static OperatorDescriptor> {
OPERATOR_TABLE
.iter()
.find(|entry| entry.spelling == spelling)
}
pub fn is_preprocessor_token(kind: TokenKind) -> bool {
matches!(kind, TokenKind::Hash | TokenKind::HashHash)
}
pub fn is_assignment_operator(kind: TokenKind) -> bool {
matches!(
kind,
TokenKind::Equal
| TokenKind::PlusEqual
| TokenKind::MinusEqual
| TokenKind::StarEqual
| TokenKind::SlashEqual
| TokenKind::PercentEqual
| TokenKind::AmpersandEqual
| TokenKind::PipeEqual
| TokenKind::CaretEqual
| TokenKind::LessLessEqual
| TokenKind::GreaterGreaterEqual
)
}
pub fn is_comparison_operator(kind: TokenKind) -> bool {
matches!(
kind,
TokenKind::Less
| TokenKind::Greater
| TokenKind::LessEqual
| TokenKind::GreaterEqual
| TokenKind::EqualEqual
| TokenKind::NotEqual
| TokenKind::Spaceship
)
}
pub fn is_binary_arithmetic_operator(kind: TokenKind) -> bool {
matches!(
kind,
TokenKind::Plus
| TokenKind::Minus
| TokenKind::Star
| TokenKind::Slash
| TokenKind::Percent
| TokenKind::Ampersand
| TokenKind::Pipe
| TokenKind::Caret
| TokenKind::LessLess
| TokenKind::GreaterGreater
)
}
pub fn is_unary_operator(kind: TokenKind) -> bool {
matches!(
kind,
TokenKind::PlusPlus
| TokenKind::MinusMinus
| TokenKind::Tilde
| TokenKind::Exclaim
| TokenKind::Star
| TokenKind::Ampersand
)
}
pub fn is_logical_operator(kind: TokenKind) -> bool {
matches!(
kind,
TokenKind::AndAnd | TokenKind::OrOr | TokenKind::Exclaim
)
}
pub fn is_bitwise_operator(kind: TokenKind) -> bool {
matches!(
kind,
TokenKind::Ampersand
| TokenKind::Pipe
| TokenKind::Caret
| TokenKind::Tilde
| TokenKind::LessLess
| TokenKind::GreaterGreater
)
}
pub fn operator_precedence(kind: TokenKind) -> u8 {
lookup_operator_by_spelling(match kind {
TokenKind::Plus => "+",
TokenKind::Minus => "-",
TokenKind::Star => "*",
TokenKind::Slash => "/",
TokenKind::Percent => "%",
TokenKind::Ampersand => "&",
TokenKind::Pipe => "|",
TokenKind::Caret => "^",
TokenKind::Tilde => "~",
TokenKind::Exclaim => "!",
TokenKind::Less => "<",
TokenKind::Greater => ">",
TokenKind::LessEqual => "<=",
TokenKind::GreaterEqual => ">=",
TokenKind::EqualEqual => "==",
TokenKind::NotEqual => "!=",
TokenKind::AndAnd => "&&",
TokenKind::OrOr => "||",
TokenKind::PlusPlus => "++",
TokenKind::MinusMinus => "--",
TokenKind::LessLess => "<<",
TokenKind::GreaterGreater => ">>",
TokenKind::PlusEqual => "+=",
TokenKind::MinusEqual => "-=",
TokenKind::StarEqual => "*=",
TokenKind::SlashEqual => "/=",
TokenKind::PercentEqual => "%=",
TokenKind::AmpersandEqual => "&=",
TokenKind::PipeEqual => "|=",
TokenKind::CaretEqual => "^=",
TokenKind::LessLessEqual => "<<=",
TokenKind::GreaterGreaterEqual => ">>=",
TokenKind::Equal => "=",
TokenKind::Comma => ",",
TokenKind::Question => "?",
TokenKind::Spaceship => "<=>",
TokenKind::Arrow => "->",
TokenKind::Dot => ".",
_ => return 0,
})
.map(|entry| entry.precedence)
.unwrap_or(0)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CppAlternativeToken {
pub keyword: &'static str,
pub kind: TokenKind,
pub operator_spelling: &'static str,
}
pub static CPP_ALTERNATIVE_TOKENS: &[CppAlternativeToken] = &[
CppAlternativeToken {
keyword: "and",
kind: TokenKind::AndAnd,
operator_spelling: "&&",
},
CppAlternativeToken {
keyword: "and_eq",
kind: TokenKind::AmpersandEqual,
operator_spelling: "&=",
},
CppAlternativeToken {
keyword: "bitand",
kind: TokenKind::Ampersand,
operator_spelling: "&",
},
CppAlternativeToken {
keyword: "bitor",
kind: TokenKind::Pipe,
operator_spelling: "|",
},
CppAlternativeToken {
keyword: "compl",
kind: TokenKind::Tilde,
operator_spelling: "~",
},
CppAlternativeToken {
keyword: "not",
kind: TokenKind::Exclaim,
operator_spelling: "!",
},
CppAlternativeToken {
keyword: "not_eq",
kind: TokenKind::NotEqual,
operator_spelling: "!=",
},
CppAlternativeToken {
keyword: "or",
kind: TokenKind::OrOr,
operator_spelling: "||",
},
CppAlternativeToken {
keyword: "or_eq",
kind: TokenKind::PipeEqual,
operator_spelling: "|=",
},
CppAlternativeToken {
keyword: "xor",
kind: TokenKind::Caret,
operator_spelling: "^",
},
CppAlternativeToken {
keyword: "xor_eq",
kind: TokenKind::CaretEqual,
operator_spelling: "^=",
},
];
pub fn is_cpp_alternative_token(ident: &str) -> Option<&'static CppAlternativeToken> {
CPP_ALTERNATIVE_TOKENS
.iter()
.find(|entry| entry.keyword == ident)
}
pub fn lookup_cpp_alternative(ident: &str) -> Option<TokenKind> {
is_cpp_alternative_token(ident).map(|entry| entry.kind)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PPDirectiveKind {
Define,
Undef,
Include,
If,
Ifdef,
Ifndef,
Elif,
Else,
Endif,
Error,
Pragma,
Line,
Warning,
Elifdef, Elifndef, Embed, IncludeNext, Import, Module, Unknown,
}
impl PPDirectiveKind {
pub fn from_str(s: &str) -> Self {
match s {
"define" => Self::Define,
"undef" => Self::Undef,
"include" => Self::Include,
"if" => Self::If,
"ifdef" => Self::Ifdef,
"ifndef" => Self::Ifndef,
"elif" => Self::Elif,
"else" => Self::Else,
"endif" => Self::Endif,
"error" => Self::Error,
"pragma" => Self::Pragma,
"line" => Self::Line,
"warning" => Self::Warning,
"elifdef" => Self::Elifdef,
"elifndef" => Self::Elifndef,
"embed" => Self::Embed,
"include_next" => Self::IncludeNext,
"import" => Self::Import,
"module" => Self::Module,
_ => Self::Unknown,
}
}
pub fn to_token_kind(self) -> TokenKind {
match self {
Self::Define => TokenKind::PPDefine,
Self::Undef => TokenKind::PPUndef,
Self::Include => TokenKind::PPInclude,
Self::If => TokenKind::PPIf,
Self::Ifdef => TokenKind::PPIfdef,
Self::Ifndef => TokenKind::PPIfndef,
Self::Elif => TokenKind::PPElif,
Self::Else => TokenKind::PPElse,
Self::Endif => TokenKind::PPEndif,
Self::Error => TokenKind::PPError,
Self::Pragma => TokenKind::PPPragma,
Self::Line => TokenKind::PPLine,
Self::Warning => TokenKind::PPWarning,
_ => TokenKind::PPDirective,
}
}
pub fn is_conditional(self) -> bool {
matches!(
self,
Self::If | Self::Ifdef | Self::Ifndef | Self::Elif | Self::Elifdef | Self::Elifndef
)
}
pub fn is_conditional_start(self) -> bool {
matches!(self, Self::If | Self::Ifdef | Self::Ifndef)
}
pub fn requires_expression(self) -> bool {
matches!(self, Self::If | Self::Elif | Self::Elifdef | Self::Elifndef)
}
}
pub fn classify_pp_token(kind: TokenKind) -> Option<PPDirectiveKind> {
match kind {
TokenKind::PPDefine => Some(PPDirectiveKind::Define),
TokenKind::PPUndef => Some(PPDirectiveKind::Undef),
TokenKind::PPInclude => Some(PPDirectiveKind::Include),
TokenKind::PPIf => Some(PPDirectiveKind::If),
TokenKind::PPIfdef => Some(PPDirectiveKind::Ifdef),
TokenKind::PPIfndef => Some(PPDirectiveKind::Ifndef),
TokenKind::PPElif => Some(PPDirectiveKind::Elif),
TokenKind::PPElse => Some(PPDirectiveKind::Else),
TokenKind::PPEndif => Some(PPDirectiveKind::Endif),
TokenKind::PPError => Some(PPDirectiveKind::Error),
TokenKind::PPPragma => Some(PPDirectiveKind::Pragma),
TokenKind::PPLine => Some(PPDirectiveKind::Line),
TokenKind::PPWarning => Some(PPDirectiveKind::Warning),
_ => None,
}
}
pub fn lookup_named_character(name: &str) -> Option<u32> {
let upper: String = name.chars().map(|c| c.to_ascii_uppercase()).collect();
let s = upper.as_str();
match s {
"NULL" => Some(0x0000),
"START OF HEADING" => Some(0x0001),
"START OF TEXT" => Some(0x0002),
"END OF TEXT" => Some(0x0003),
"END OF TRANSMISSION" => Some(0x0004),
"ENQUIRY" => Some(0x0005),
"ACKNOWLEDGE" => Some(0x0006),
"BELL" | "ALERT" => Some(0x0007),
"BACKSPACE" => Some(0x0008),
"CHARACTER TABULATION" | "HORIZONTAL TABULATION" | "TAB" => Some(0x0009),
"LINE FEED" | "NEW LINE" | "NEWLINE" | "END OF LINE" => Some(0x000A),
"VERTICAL TABULATION" | "VERTICAL TAB" => Some(0x000B),
"FORM FEED" => Some(0x000C),
"CARRIAGE RETURN" => Some(0x000D),
"SHIFT OUT" => Some(0x000E),
"SHIFT IN" => Some(0x000F),
"DATA LINK ESCAPE" => Some(0x0010),
"DEVICE CONTROL ONE" => Some(0x0011),
"DEVICE CONTROL TWO" => Some(0x0012),
"DEVICE CONTROL THREE" => Some(0x0013),
"DEVICE CONTROL FOUR" => Some(0x0014),
"NEGATIVE ACKNOWLEDGE" => Some(0x0015),
"SYNCHRONOUS IDLE" => Some(0x0016),
"END OF TRANSMISSION BLOCK" => Some(0x0017),
"CANCEL" => Some(0x0018),
"END OF MEDIUM" => Some(0x0019),
"SUBSTITUTE" => Some(0x001A),
"ESCAPE" => Some(0x001B),
"INFORMATION SEPARATOR FOUR" | "FILE SEPARATOR" => Some(0x001C),
"INFORMATION SEPARATOR THREE" | "GROUP SEPARATOR" => Some(0x001D),
"INFORMATION SEPARATOR TWO" | "RECORD SEPARATOR" => Some(0x001E),
"INFORMATION SEPARATOR ONE" | "UNIT SEPARATOR" => Some(0x001F),
"DELETE" => Some(0x007F),
"SPACE" => Some(0x0020),
"EXCLAMATION MARK" => Some(0x0021),
"QUOTATION MARK" => Some(0x0022),
"NUMBER SIGN" => Some(0x0023),
"DOLLAR SIGN" => Some(0x0024),
"PERCENT SIGN" => Some(0x0025),
"AMPERSAND" => Some(0x0026),
"APOSTROPHE" => Some(0x0027),
"LEFT PARENTHESIS" => Some(0x0028),
"RIGHT PARENTHESIS" => Some(0x0029),
"ASTERISK" => Some(0x002A),
"PLUS SIGN" => Some(0x002B),
"COMMA" => Some(0x002C),
"HYPHEN-MINUS" => Some(0x002D),
"FULL STOP" | "PERIOD" => Some(0x002E),
"SOLIDUS" | "SLASH" => Some(0x002F),
"COLON" => Some(0x003A),
"SEMICOLON" => Some(0x003B),
"LESS-THAN SIGN" => Some(0x003C),
"EQUALS SIGN" => Some(0x003D),
"GREATER-THAN SIGN" => Some(0x003E),
"QUESTION MARK" => Some(0x003F),
"COMMERCIAL AT" => Some(0x0040),
"LEFT SQUARE BRACKET" => Some(0x005B),
"REVERSE SOLIDUS" | "BACKSLASH" => Some(0x005C),
"RIGHT SQUARE BRACKET" => Some(0x005D),
"CIRCUMFLEX ACCENT" => Some(0x005E),
"LOW LINE" | "UNDERSCORE" => Some(0x005F),
"GRAVE ACCENT" => Some(0x0060),
"LEFT CURLY BRACKET" | "LEFT BRACE" => Some(0x007B),
"VERTICAL LINE" | "VERTICAL BAR" => Some(0x007C),
"RIGHT CURLY BRACKET" | "RIGHT BRACE" => Some(0x007D),
"TILDE" => Some(0x007E),
"EURO SIGN" => Some(0x20AC),
"POUND SIGN" => Some(0x00A3),
"YEN SIGN" => Some(0x00A5),
"CENT SIGN" => Some(0x00A2),
"COPYRIGHT SIGN" => Some(0x00A9),
"REGISTERED SIGN" => Some(0x00AE),
"TRADE MARK SIGN" => Some(0x2122),
"DEGREE SIGN" => Some(0x00B0),
"MICRO SIGN" => Some(0x00B5),
"PILCROW SIGN" | "PARAGRAPH SIGN" => Some(0x00B6),
"MIDDLE DOT" => Some(0x00B7),
"MULTIPLICATION SIGN" => Some(0x00D7),
"DIVISION SIGN" => Some(0x00F7),
"PLUS-MINUS SIGN" => Some(0x00B1),
"NOT SIGN" => Some(0x00AC),
"BROKEN BAR" => Some(0x00A6),
"GREEK CAPITAL LETTER ALPHA" => Some(0x0391),
"GREEK CAPITAL LETTER BETA" => Some(0x0392),
"GREEK CAPITAL LETTER GAMMA" => Some(0x0393),
"GREEK CAPITAL LETTER DELTA" => Some(0x0394),
"GREEK CAPITAL LETTER EPSILON" => Some(0x0395),
"GREEK CAPITAL LETTER PI" => Some(0x03A0),
"GREEK CAPITAL LETTER SIGMA" => Some(0x03A3),
"GREEK CAPITAL LETTER OMEGA" => Some(0x03A9),
"GREEK SMALL LETTER ALPHA" => Some(0x03B1),
"GREEK SMALL LETTER BETA" => Some(0x03B2),
"GREEK SMALL LETTER GAMMA" => Some(0x03B3),
"GREEK SMALL LETTER DELTA" => Some(0x03B4),
"GREEK SMALL LETTER EPSILON" => Some(0x03B5),
"GREEK SMALL LETTER PI" => Some(0x03C0),
"GREEK SMALL LETTER SIGMA" => Some(0x03C3),
"GREEK SMALL LETTER OMEGA" => Some(0x03C9),
"FOR ALL" => Some(0x2200),
"THERE EXISTS" => Some(0x2203),
"EMPTY SET" => Some(0x2205),
"INCREMENT" | "DELTA" => Some(0x2206),
"ELEMENT OF" => Some(0x2208),
"NOT AN ELEMENT OF" => Some(0x2209),
"CONTAINS AS MEMBER" => Some(0x220B),
"SUBSET OF" => Some(0x2282),
"SUPERSET OF" => Some(0x2283),
"INTERSECTION" => Some(0x2229),
"UNION" => Some(0x222A),
"INTEGRAL" => Some(0x222B),
"PARTIAL DIFFERENTIAL" => Some(0x2202),
"NABLA" => Some(0x2207),
"INFINITY" => Some(0x221E),
"ALMOST EQUAL TO" | "APPROXIMATELY EQUAL TO" => Some(0x2248),
"NOT EQUAL TO" => Some(0x2260),
"IDENTICAL TO" => Some(0x2261),
"LESS-THAN OR EQUAL TO" => Some(0x2264),
"GREATER-THAN OR EQUAL TO" => Some(0x2265),
"SQUARE ROOT" => Some(0x221A),
"CUBE ROOT" => Some(0x221B),
"SUM" | "N-ARY SUMMATION" => Some(0x2211),
"PRODUCT" | "N-ARY PRODUCT" => Some(0x220F),
"LEFTWARDS ARROW" => Some(0x2190),
"UPWARDS ARROW" => Some(0x2191),
"RIGHTWARDS ARROW" => Some(0x2192),
"DOWNWARDS ARROW" => Some(0x2193),
"LEFT RIGHT ARROW" => Some(0x2194),
_ => None,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TokenCategory {
Keyword,
Identifier,
NumericLiteral,
CharLiteral,
StringLiteral,
Operator,
Preprocessor,
Whitespace,
Eof,
Unknown,
}
pub fn classify_token(kind: TokenKind) -> TokenCategory {
match kind {
TokenKind::KwAuto
| TokenKind::KwBreak
| TokenKind::KwCase
| TokenKind::KwChar
| TokenKind::KwConst
| TokenKind::KwContinue
| TokenKind::KwDefault
| TokenKind::KwDo
| TokenKind::KwDouble
| TokenKind::KwElse
| TokenKind::KwEnum
| TokenKind::KwExtern
| TokenKind::KwFloat
| TokenKind::KwFor
| TokenKind::KwGoto
| TokenKind::KwIf
| TokenKind::KwInline
| TokenKind::KwInt
| TokenKind::KwLong
| TokenKind::KwRegister
| TokenKind::KwRestrict
| TokenKind::KwReturn
| TokenKind::KwShort
| TokenKind::KwSigned
| TokenKind::KwSizeof
| TokenKind::KwStatic
| TokenKind::KwStruct
| TokenKind::KwSwitch
| TokenKind::KwTypedef
| TokenKind::KwUnion
| TokenKind::KwUnsigned
| TokenKind::KwVoid
| TokenKind::KwVolatile
| TokenKind::KwWhile
| TokenKind::KwBool
| TokenKind::KwComplex
| TokenKind::KwImaginary
| TokenKind::KwAlignas
| TokenKind::KwAlignof
| TokenKind::KwAtomic
| TokenKind::KwGeneric
| TokenKind::KwNoreturn
| TokenKind::KwStaticAssert
| TokenKind::KwThreadLocal
| TokenKind::KwCatch
| TokenKind::KwChar8T
| TokenKind::KwChar16T
| TokenKind::KwChar32T
| TokenKind::KwClass
| TokenKind::KwConcept
| TokenKind::KwConstCast
| TokenKind::KwConstexpr
| TokenKind::KwCoAwait
| TokenKind::KwCoReturn
| TokenKind::KwCoYield
| TokenKind::KwDecltype
| TokenKind::KwDelete
| TokenKind::KwDynamicCast
| TokenKind::KwExplicit
| TokenKind::KwExport
| TokenKind::KwFalse
| TokenKind::KwFriend
| TokenKind::KwMutable
| TokenKind::KwNamespace
| TokenKind::KwNew
| TokenKind::KwNoexcept
| TokenKind::KwNullptr
| TokenKind::KwOperator
| TokenKind::KwPrivate
| TokenKind::KwProtected
| TokenKind::KwPublic
| TokenKind::KwReinterpretCast
| TokenKind::KwRequires
| TokenKind::KwStaticCast
| TokenKind::KwTemplate
| TokenKind::KwThis
| TokenKind::KwThrow
| TokenKind::KwTrue
| TokenKind::KwTry
| TokenKind::KwTypeid
| TokenKind::KwTypename
| TokenKind::KwUsing
| TokenKind::KwVirtual
| TokenKind::KwWcharT
| TokenKind::KwAnd
| TokenKind::KwAndEq
| TokenKind::KwBitAnd
| TokenKind::KwBitor
| TokenKind::KwCompl
| TokenKind::KwNot
| TokenKind::KwNotEq
| TokenKind::KwOr
| TokenKind::KwOrEq
| TokenKind::KwXor
| TokenKind::KwXorEq => TokenCategory::Keyword,
TokenKind::Identifier => TokenCategory::Identifier,
TokenKind::NumericLiteral => TokenCategory::NumericLiteral,
TokenKind::CharLiteral
| TokenKind::WideCharLiteral
| TokenKind::UTF8CharLiteral
| TokenKind::UTF16CharLiteral
| TokenKind::UTF32CharLiteral => TokenCategory::CharLiteral,
TokenKind::StringLiteral
| TokenKind::WideStringLiteral
| TokenKind::UTF8StringLiteral
| TokenKind::UTF16StringLiteral
| TokenKind::UTF32StringLiteral
| TokenKind::RawStringLiteral => TokenCategory::StringLiteral,
TokenKind::LParen
| TokenKind::RParen
| TokenKind::LBrace
| TokenKind::RBrace
| TokenKind::LBracket
| TokenKind::RBracket
| TokenKind::Dot
| TokenKind::Arrow
| TokenKind::PlusPlus
| TokenKind::MinusMinus
| TokenKind::Plus
| TokenKind::Minus
| TokenKind::Star
| TokenKind::Slash
| TokenKind::Percent
| TokenKind::Ampersand
| TokenKind::Pipe
| TokenKind::Caret
| TokenKind::Tilde
| TokenKind::Exclaim
| TokenKind::Less
| TokenKind::Greater
| TokenKind::LessEqual
| TokenKind::GreaterEqual
| TokenKind::EqualEqual
| TokenKind::NotEqual
| TokenKind::AndAnd
| TokenKind::OrOr
| TokenKind::LessLess
| TokenKind::GreaterGreater
| TokenKind::PlusEqual
| TokenKind::MinusEqual
| TokenKind::StarEqual
| TokenKind::SlashEqual
| TokenKind::PercentEqual
| TokenKind::AmpersandEqual
| TokenKind::PipeEqual
| TokenKind::CaretEqual
| TokenKind::LessLessEqual
| TokenKind::GreaterGreaterEqual
| TokenKind::Equal
| TokenKind::Semicolon
| TokenKind::Comma
| TokenKind::Colon
| TokenKind::Question
| TokenKind::Ellipsis
| TokenKind::ArrowStar
| TokenKind::DotStar
| TokenKind::ScopeResolution
| TokenKind::Spaceship => TokenCategory::Operator,
TokenKind::PPDirective
| TokenKind::PPDefine
| TokenKind::PPUndef
| TokenKind::PPIf
| TokenKind::PPIfdef
| TokenKind::PPIfndef
| TokenKind::PPElif
| TokenKind::PPElse
| TokenKind::PPEndif
| TokenKind::PPInclude
| TokenKind::PPError
| TokenKind::PPPragma
| TokenKind::PPLine
| TokenKind::PPWarning
| TokenKind::Hash
| TokenKind::HashHash => TokenCategory::Preprocessor,
TokenKind::Newline | TokenKind::Comment => TokenCategory::Whitespace,
TokenKind::Eof => TokenCategory::Eof,
TokenKind::Unknown | TokenKind::UserDefinedLiteral => TokenCategory::Unknown,
_ => TokenCategory::Unknown,
}
}
pub fn reconstruct_numeric_literal(result: &NumericLiteralResult) -> String {
let mut s = String::new();
match result.radix {
NumericRadix::Binary => s.push_str("0b"),
NumericRadix::Hex => s.push_str("0x"),
NumericRadix::Octal => {
if !result.significand.starts_with('0') {
s.push('0');
}
}
NumericRadix::Decimal => {}
}
s.push_str(&result.significand);
if result.has_decimal_point {
s.push('.');
}
if let Some(exp) = result.exponent {
match result.radix {
NumericRadix::Hex | NumericRadix::Binary => s.push('p'),
_ => s.push('e'),
}
if exp < 0 {
s.push('-');
s.push_str(&(-exp).to_string());
} else {
s.push('+');
s.push_str(&exp.to_string());
}
}
match result.float_suffix {
FloatSuffixKind::Float => s.push('f'),
FloatSuffixKind::LongDouble => s.push('L'),
FloatSuffixKind::Float16 => s.push_str("f16"),
FloatSuffixKind::Float32 => s.push_str("f32"),
FloatSuffixKind::Float64 => s.push_str("f64"),
FloatSuffixKind::Float128 => s.push_str("f128"),
FloatSuffixKind::BFloat16 => s.push_str("bf16"),
FloatSuffixKind::ExplicitDouble => s.push('d'),
FloatSuffixKind::Decimal32 => s.push_str("df"),
FloatSuffixKind::Decimal64 => s.push_str("dd"),
FloatSuffixKind::Decimal128 => s.push_str("dl"),
FloatSuffixKind::Float80 => s.push('w'),
FloatSuffixKind::Float128Quad => s.push('q'),
FloatSuffixKind::Imaginary => s.push('i'),
FloatSuffixKind::FloatImaginary => s.push_str("fi"),
FloatSuffixKind::LongDoubleImaginary => s.push_str("li"),
FloatSuffixKind::None => {}
}
match result.int_suffix {
IntSuffixKind::None => {}
IntSuffixKind::Unsigned => s.push('u'),
IntSuffixKind::Long => s.push('l'),
IntSuffixKind::UnsignedLong => s.push_str("ul"),
IntSuffixKind::LongLong => s.push_str("ll"),
IntSuffixKind::UnsignedLongLong => s.push_str("ull"),
IntSuffixKind::BitPrecise => s.push_str("wb"),
IntSuffixKind::UnsignedBitPrecise => s.push_str("uwb"),
IntSuffixKind::Size => s.push('z'),
IntSuffixKind::UnsignedSize => s.push_str("uz"),
IntSuffixKind::Int64 => s.push_str("i64"),
IntSuffixKind::Uint64 => s.push_str("u64"),
IntSuffixKind::Int128 => s.push_str("i128"),
IntSuffixKind::Uint128 => s.push_str("u128"),
IntSuffixKind::BitInt(w) => {
s.push_str(&format!("_BitInt({})", w));
}
IntSuffixKind::UnsignedBitInt(w) => {
s.push_str(&format!("unsigned _BitInt({})", w));
}
IntSuffixKind::Imaginary => s.push('i'),
IntSuffixKind::UnsignedImaginary => s.push_str("ui"),
IntSuffixKind::LongImaginary => s.push_str("li"),
}
s
}
pub fn reconstruct_char_literal(parser: &CharLiteralParser) -> String {
let mut s = String::new();
match parser.prefix {
CharPrefixKind::Wide => s.push('L'),
CharPrefixKind::Utf8 => s.push_str("u8"),
CharPrefixKind::Utf16 => s.push('u'),
CharPrefixKind::Utf32 => s.push('U'),
CharPrefixKind::Narrow => {}
}
s.push('\'');
if parser.is_multi_char {
for &byte in &parser.multi_char_values {
if byte == b'\\' || byte == b'\'' {
s.push('\\');
}
if (0x20..=0x7E).contains(&byte) {
s.push(byte as char);
} else {
s.push_str(&format!("\\x{:02X}", byte));
}
}
} else {
let cp = parser.value;
match cp {
0x07 => s.push_str("\\a"),
0x08 => s.push_str("\\b"),
0x0C => s.push_str("\\f"),
0x0A => s.push_str("\\n"),
0x0D => s.push_str("\\r"),
0x09 => s.push_str("\\t"),
0x0B => s.push_str("\\v"),
0x5C => s.push_str("\\\\"),
0x27 => s.push_str("\\'"),
0x22 => s.push_str("\\\""),
0x00 => s.push_str("\\0"),
0x1B => s.push_str("\\e"),
_ => {
if (0x20..=0x7E).contains(&cp) && cp != b'\\' as u32 && cp != b'\'' as u32 {
if let Some(ch) = char::from_u32(cp) {
s.push(ch);
} else {
s.push_str(&format!("\\x{:X}", cp));
}
} else if cp <= 0xFF {
s.push_str(&format!("\\x{:02X}", cp));
} else if cp <= 0xFFFF {
s.push_str(&format!("\\u{:04X}", cp));
} else {
s.push_str(&format!("\\U{:08X}", cp));
}
}
}
}
s.push('\'');
s
}
#[inline]
pub fn needs_escape_in_string(ch: char) -> bool {
matches!(ch, '"' | '\\' | '\n' | '\r' | '\t' | '\0') || ch.is_control()
}
pub fn escape_for_string(ch: char) -> String {
match ch {
'"' => "\\\"".to_string(),
'\\' => "\\\\".to_string(),
'\n' => "\\n".to_string(),
'\r' => "\\r".to_string(),
'\t' => "\\t".to_string(),
'\0' => "\\0".to_string(),
_ if ch.is_control() => format!("\\x{:02X}", ch as u32),
_ => ch.to_string(),
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CFeature {
LongLong,
Bool,
Complex,
LineComments,
HexFloat,
BinaryLiterals,
DigitSeparators,
BitInt,
IeeeFloatSuffixes,
Utf8LiteralPrefix,
UnicodeLiteralPrefixes,
Trigraphs,
Digraphs,
RawStrings,
UcnInIdentifiers,
FixedWidthSuffixes,
ImaginaryLiterals,
NamedEscapes,
SizeSuffix,
BitPreciseSuffix,
}
impl CFeature {
pub fn is_available_in(self, standard: CLangStandard) -> bool {
match self {
Self::LongLong => matches!(
standard,
CLangStandard::C99
| CLangStandard::C11
| CLangStandard::C17
| CLangStandard::C23
| CLangStandard::Gnu99
| CLangStandard::Gnu11
| CLangStandard::Gnu17
),
Self::Bool => matches!(
standard,
CLangStandard::C99
| CLangStandard::C11
| CLangStandard::C17
| CLangStandard::C23
| CLangStandard::Gnu99
| CLangStandard::Gnu11
| CLangStandard::Gnu17
),
Self::Complex => matches!(
standard,
CLangStandard::C99
| CLangStandard::C11
| CLangStandard::C17
| CLangStandard::C23
| CLangStandard::Gnu99
| CLangStandard::Gnu11
| CLangStandard::Gnu17
),
Self::LineComments => matches!(
standard,
CLangStandard::C99
| CLangStandard::C11
| CLangStandard::C17
| CLangStandard::C23
| CLangStandard::Gnu99
| CLangStandard::Gnu11
| CLangStandard::Gnu17
),
Self::HexFloat => matches!(
standard,
CLangStandard::C99
| CLangStandard::C11
| CLangStandard::C17
| CLangStandard::C23
| CLangStandard::Gnu99
| CLangStandard::Gnu11
| CLangStandard::Gnu17
),
Self::BinaryLiterals => matches!(
standard,
CLangStandard::C23
| CLangStandard::Gnu89
| CLangStandard::Gnu99
| CLangStandard::Gnu11
| CLangStandard::Gnu17
),
Self::DigitSeparators => matches!(standard, CLangStandard::C23),
Self::BitInt => matches!(standard, CLangStandard::C23),
Self::IeeeFloatSuffixes => matches!(standard, CLangStandard::C23),
Self::Utf8LiteralPrefix => matches!(standard, CLangStandard::C23),
Self::UnicodeLiteralPrefixes => matches!(
standard,
CLangStandard::C11
| CLangStandard::C17
| CLangStandard::C23
| CLangStandard::Gnu11
| CLangStandard::Gnu17
),
Self::Trigraphs => !matches!(standard, CLangStandard::C23),
Self::Digraphs => true, Self::RawStrings => matches!(standard, CLangStandard::C23),
Self::UcnInIdentifiers => matches!(
standard,
CLangStandard::C99
| CLangStandard::C11
| CLangStandard::C17
| CLangStandard::C23
| CLangStandard::Gnu99
| CLangStandard::Gnu11
| CLangStandard::Gnu17
),
Self::FixedWidthSuffixes => standard.is_gnu(),
Self::ImaginaryLiterals => standard.is_gnu(),
Self::NamedEscapes => matches!(standard, CLangStandard::C23),
Self::SizeSuffix => matches!(standard, CLangStandard::C23),
Self::BitPreciseSuffix => matches!(standard, CLangStandard::C23),
}
}
pub fn description(self) -> &'static str {
match self {
Self::LongLong => "long long integer type",
Self::Bool => "_Bool type",
Self::Complex => "_Complex and _Imaginary types",
Self::LineComments => "// single-line comments",
Self::HexFloat => "hexadecimal floating-point literals",
Self::BinaryLiterals => "0b/0B binary integer literals",
Self::DigitSeparators => "digit separators (single quote)",
Self::BitInt => "_BitInt(N) bit-precise integers",
Self::IeeeFloatSuffixes => "IEEE interchange float suffixes (f16, f32, f64, f128)",
Self::Utf8LiteralPrefix => "u8 character/string literal prefix",
Self::UnicodeLiteralPrefixes => {
"u/U character/string literal prefixes (char16_t/char32_t)"
}
Self::Trigraphs => "trigraph sequences (??=, ??(, etc.)",
Self::Digraphs => "digraph alternative tokens (<:, :>, etc.)",
Self::RawStrings => "raw string literals R\"(...)\"",
Self::UcnInIdentifiers => "universal character names in identifiers",
Self::FixedWidthSuffixes => "fixed-width integer suffixes (i64, i128, u64, u128)",
Self::ImaginaryLiterals => "imaginary literal suffixes (i, j)",
Self::NamedEscapes => "named escape sequences \\N{...}",
Self::SizeSuffix => "size_t literal suffix (z/Z)",
Self::BitPreciseSuffix => "bit-precise integer suffix (wb/WB)",
}
}
}
pub fn features_for_standard(standard: CLangStandard) -> Vec<CFeature> {
let all = [
CFeature::LongLong,
CFeature::Bool,
CFeature::Complex,
CFeature::LineComments,
CFeature::HexFloat,
CFeature::BinaryLiterals,
CFeature::DigitSeparators,
CFeature::BitInt,
CFeature::IeeeFloatSuffixes,
CFeature::Utf8LiteralPrefix,
CFeature::UnicodeLiteralPrefixes,
CFeature::Trigraphs,
CFeature::Digraphs,
CFeature::RawStrings,
CFeature::UcnInIdentifiers,
CFeature::FixedWidthSuffixes,
CFeature::ImaginaryLiterals,
CFeature::NamedEscapes,
CFeature::SizeSuffix,
CFeature::BitPreciseSuffix,
];
all.iter()
.filter(|f| f.is_available_in(standard))
.copied()
.collect()
}
pub struct DeepLexer {
pub standard: CLangStandard,
}
impl DeepLexer {
pub fn new(standard: CLangStandard) -> Self {
Self { standard }
}
pub fn lex_numeric_literal(&self, source: &str) -> NumericLiteralResult {
parse_numeric_literal(source, self.standard)
}
pub fn lex_char_literal(&self, source: &str) -> CharLiteralParser {
parse_char_literal(source)
}
pub fn lex_string_literal(&self, source: &str) -> StringLiteralParser {
parse_string_literal(source)
}
pub fn lex_concatenated_strings(&self, segments: &[&str]) -> StringLiteralParser {
parse_and_concatenate_strings(segments)
}
pub fn lex_ucn(&self, source: &str) -> UcnParser {
parse_ucn(source)
}
pub fn lex_ucn_for_identifier(&self, source: &str) -> UcnParser {
parse_ucn_for_identifier(source)
}
pub fn lex_operator(&self, source: &str) -> Option<(TokenKind, usize)> {
try_lex_operator(source)
}
pub fn lex_digraph(&self, source: &str) -> Option<(TokenKind, usize)> {
is_digraph(source)
}
pub fn lex_trigraph(&self, source: &str) -> Option<(char, usize)> {
is_trigraph(source)
}
pub fn replace_trigraphs(&self, source: &str) -> String {
replace_trigraphs(source)
}
pub fn has_feature(&self, feature: CFeature) -> bool {
feature.is_available_in(self.standard)
}
pub fn available_features(&self) -> Vec<CFeature> {
features_for_standard(self.standard)
}
pub fn classify(&self, kind: TokenKind) -> TokenCategory {
classify_token(kind)
}
pub fn reconstruct_numeric(&self, result: &NumericLiteralResult) -> String {
reconstruct_numeric_literal(result)
}
pub fn reconstruct_char(&self, parser: &CharLiteralParser) -> String {
reconstruct_char_literal(parser)
}
}
impl Default for DeepLexer {
fn default() -> Self {
Self::new(CLangStandard::C17)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_decimal_integer() {
let result = parse_numeric_literal("42", CLangStandard::C17);
assert!(!result.had_error);
assert!(!result.is_floating());
assert_eq!(result.radix, NumericRadix::Decimal);
assert_eq!(result.significand, "42");
assert_eq!(result.int_suffix, IntSuffixKind::None);
}
#[test]
fn test_parse_decimal_integer_with_unsigned_suffix() {
let result = parse_numeric_literal("42u", CLangStandard::C99);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::Unsigned);
assert!(result.int_suffix.is_unsigned());
}
#[test]
fn test_parse_decimal_integer_with_long_suffix() {
let result = parse_numeric_literal("100L", CLangStandard::C99);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::Long);
assert!(result.int_suffix.is_long());
}
#[test]
fn test_parse_decimal_integer_with_long_long_suffix() {
let result = parse_numeric_literal("9999999999LL", CLangStandard::C99);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::LongLong);
assert!(result.int_suffix.is_long_long());
}
#[test]
fn test_parse_decimal_integer_with_unsigned_long_long() {
let result = parse_numeric_literal("0xFFFFFFFFFFFFFFFFull", CLangStandard::C99);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::UnsignedLongLong);
assert!(result.int_suffix.is_unsigned());
assert!(result.int_suffix.is_long_long());
}
#[test]
fn test_parse_hex_integer() {
let result = parse_numeric_literal("0xDEADBEEF", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.radix, NumericRadix::Hex);
assert_eq!(result.significand, "DEADBEEF");
}
#[test]
fn test_parse_hex_integer_lowercase_prefix() {
let result = parse_numeric_literal("0XABCD", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.radix, NumericRadix::Hex);
}
#[test]
fn test_parse_binary_integer() {
let result = parse_numeric_literal("0b10101010", CLangStandard::C23);
assert!(!result.had_error);
assert_eq!(result.radix, NumericRadix::Binary);
assert_eq!(result.significand, "10101010");
}
#[test]
fn test_parse_binary_integer_uppercase_prefix() {
let result = parse_numeric_literal("0B11110000", CLangStandard::C23);
assert!(!result.had_error);
assert_eq!(result.radix, NumericRadix::Binary);
assert_eq!(result.significand, "11110000");
}
#[test]
fn test_parse_octal_integer() {
let result = parse_numeric_literal("0777", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.radix, NumericRadix::Octal);
}
#[test]
fn test_parse_decimal_zero() {
let result = parse_numeric_literal("0", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.radix, NumericRadix::Decimal);
}
#[test]
fn test_parse_decimal_float() {
let result = parse_numeric_literal("3.14159", CLangStandard::C17);
assert!(!result.had_error);
assert!(result.is_floating());
assert_eq!(result.radix, NumericRadix::Decimal);
assert!(result.has_decimal_point);
}
#[test]
fn test_parse_decimal_float_with_exponent() {
let result = parse_numeric_literal("1.5e10", CLangStandard::C17);
assert!(!result.had_error);
assert!(result.is_floating());
assert_eq!(result.exponent, Some(10));
assert!(!result.exponent_is_negative);
}
#[test]
fn test_parse_decimal_float_negative_exponent() {
let result = parse_numeric_literal("2.5E-3", CLangStandard::C17);
assert!(!result.had_error);
assert!(result.is_floating());
assert_eq!(result.exponent, Some(-3));
}
#[test]
fn test_parse_hex_float() {
let result = parse_numeric_literal("0x1.FFFFp+10", CLangStandard::C17);
assert!(!result.had_error);
assert!(result.is_floating());
assert_eq!(result.radix, NumericRadix::Hex);
assert_eq!(result.exponent, Some(10));
assert!(result.has_decimal_point);
}
#[test]
fn test_parse_hex_float_negative_exponent() {
let result = parse_numeric_literal("0x1.0p-3", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.exponent, Some(-3));
}
#[test]
fn test_parse_float_with_f_suffix() {
let result = parse_numeric_literal("1.0f", CLangStandard::C17);
assert!(!result.had_error);
assert!(result.is_floating());
assert_eq!(result.float_suffix, FloatSuffixKind::Float);
}
#[test]
fn test_parse_float_with_long_double_suffix() {
let result = parse_numeric_literal("3.14L", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.float_suffix, FloatSuffixKind::LongDouble);
}
#[test]
fn test_parse_integer_with_c23_size_suffix() {
let result = parse_numeric_literal("100z", CLangStandard::C23);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::Size);
assert!(result.int_suffix.is_c23());
}
#[test]
fn test_parse_integer_with_c23_unsigned_size_suffix() {
let result = parse_numeric_literal("200uz", CLangStandard::C23);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::UnsignedSize);
assert!(result.int_suffix.is_c23());
assert!(result.int_suffix.is_unsigned());
}
#[test]
fn test_parse_integer_with_c23_bitprecise_suffix() {
let result = parse_numeric_literal("42wb", CLangStandard::C23);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::BitPrecise);
assert!(result.int_suffix.is_bit_precise());
}
#[test]
fn test_parse_leading_decimal_point_float() {
let result = parse_numeric_literal(".5", CLangStandard::C17);
assert!(!result.had_error);
assert!(result.is_floating());
assert!(result.has_decimal_point);
}
#[test]
fn test_parse_trailing_decimal_point_float() {
let result = parse_numeric_literal("1.", CLangStandard::C17);
assert!(!result.had_error);
assert!(result.is_floating());
assert!(result.has_decimal_point);
}
#[test]
fn test_parse_i64_suffix() {
let result = parse_numeric_literal("42i64", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::Int64);
assert!(result.int_suffix.is_fixed_width());
}
#[test]
fn test_parse_u64_suffix() {
let result = parse_numeric_literal("42u64", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::Uint64);
}
#[test]
fn test_parse_i128_suffix() {
let result = parse_numeric_literal("42i128", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::Int128);
}
#[test]
fn test_parse_u128_suffix() {
let result = parse_numeric_literal("42u128", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::Uint128);
}
#[test]
fn test_parse_digit_separator_in_decimal() {
let result = parse_numeric_literal("1'000'000", CLangStandard::C23);
assert!(!result.had_error);
assert!(result.had_digit_separators);
assert_eq!(result.significand, "1000000");
}
#[test]
fn test_parse_digit_separator_in_hex() {
let result = parse_numeric_literal("0xFF'FF'FF'FF", CLangStandard::C23);
assert!(!result.had_error);
assert!(result.had_digit_separators);
}
#[test]
fn test_parse_digit_separator_in_binary() {
let result = parse_numeric_literal("0b1111'0000'1010'0101", CLangStandard::C23);
assert!(!result.had_error);
assert!(result.had_digit_separators);
}
#[test]
fn test_parse_c23_float16_suffix() {
let result = parse_numeric_literal("1.0f16", CLangStandard::C23);
assert!(!result.had_error);
assert_eq!(result.float_suffix, FloatSuffixKind::Float16);
}
#[test]
fn test_parse_c23_float32_suffix() {
let result = parse_numeric_literal("1.0f32", CLangStandard::C23);
assert!(!result.had_error);
assert_eq!(result.float_suffix, FloatSuffixKind::Float32);
}
#[test]
fn test_parse_c23_float64_suffix() {
let result = parse_numeric_literal("1.0f64", CLangStandard::C23);
assert!(!result.had_error);
assert_eq!(result.float_suffix, FloatSuffixKind::Float64);
}
#[test]
fn test_parse_c23_float128_suffix() {
let result = parse_numeric_literal("1.0f128", CLangStandard::C23);
assert!(!result.had_error);
assert_eq!(result.float_suffix, FloatSuffixKind::Float128);
}
#[test]
fn test_parse_c23_bf16_suffix() {
let result = parse_numeric_literal("1.0bf16", CLangStandard::C23);
assert!(!result.had_error);
assert_eq!(result.float_suffix, FloatSuffixKind::BFloat16);
}
#[test]
fn test_parse_narrow_char_literal() {
let parser = parse_char_literal("'a'");
assert!(!parser.had_error);
assert_eq!(parser.prefix, CharPrefixKind::Narrow);
assert_eq!(parser.value, 'a' as u32);
assert!(!parser.is_multi_char);
assert!(parser.is_complete);
}
#[test]
fn test_parse_narrow_char_escape_newline() {
let parser = parse_char_literal("'\\n'");
assert!(!parser.had_error);
assert_eq!(parser.value, 0x0A);
}
#[test]
fn test_parse_narrow_char_escape_tab() {
let parser = parse_char_literal("'\\t'");
assert!(!parser.had_error);
assert_eq!(parser.value, 0x09);
}
#[test]
fn test_parse_narrow_char_escape_backslash() {
let parser = parse_char_literal("'\\\\'");
assert!(!parser.had_error);
assert_eq!(parser.value, 0x5C);
}
#[test]
fn test_parse_narrow_char_escape_single_quote() {
let parser = parse_char_literal("'\\''");
assert!(!parser.had_error);
assert_eq!(parser.value, 0x27);
}
#[test]
fn test_parse_narrow_char_octal_escape() {
let parser = parse_char_literal("'\\0'");
assert!(!parser.had_error);
assert_eq!(parser.value, 0);
}
#[test]
fn test_parse_narrow_char_octal_escape_three_digit() {
let parser = parse_char_literal("'\\077'");
assert!(!parser.had_error);
assert_eq!(parser.value, 0o77);
}
#[test]
fn test_parse_narrow_char_hex_escape() {
let parser = parse_char_literal("'\\x41'");
assert!(!parser.had_error);
assert_eq!(parser.value, 0x41);
}
#[test]
fn test_parse_narrow_char_hex_escape_long() {
let parser = parse_char_literal("'\\x00FF'");
assert!(!parser.had_error);
assert!(parser.had_error || parser.value <= 0xFF);
}
#[test]
fn test_parse_wide_char_literal() {
let parser = parse_char_literal("L'a'");
assert!(!parser.had_error);
assert_eq!(parser.prefix, CharPrefixKind::Wide);
assert_eq!(parser.value, 'a' as u32);
}
#[test]
fn test_parse_utf16_char_literal() {
let parser = parse_char_literal("u'a'");
assert!(!parser.had_error);
assert_eq!(parser.prefix, CharPrefixKind::Utf16);
}
#[test]
fn test_parse_utf32_char_literal() {
let parser = parse_char_literal("U'a'");
assert!(!parser.had_error);
assert_eq!(parser.prefix, CharPrefixKind::Utf32);
}
#[test]
fn test_parse_utf8_char_literal() {
let parser = parse_char_literal("u8'a'");
assert!(!parser.had_error);
assert_eq!(parser.prefix, CharPrefixKind::Utf8);
}
#[test]
fn test_parse_ucn_short_in_char() {
let parser = parse_char_literal("'\\u03A9'");
assert!(!parser.had_error);
assert_eq!(parser.value, 0x03A9); }
#[test]
fn test_parse_ucn_long_in_char() {
let parser = parse_char_literal("U'\\U0001F600'");
assert!(!parser.had_error);
assert_eq!(parser.prefix, CharPrefixKind::Utf32);
assert_eq!(parser.value, 0x1F600); }
#[test]
fn test_parse_multi_char_literal() {
let parser = parse_char_literal("'ab'");
assert!(!parser.had_error);
assert!(parser.is_multi_char);
assert_eq!(parser.multi_char_values.len(), 2);
}
#[test]
fn test_parse_narrow_string() {
let parser = parse_string_literal("\"hello\"");
assert!(!parser.had_error);
assert_eq!(parser.prefix, StringPrefixKind::Narrow);
assert_eq!(parser.string_bytes, b"hello".to_vec());
assert!(parser.is_complete);
}
#[test]
fn test_parse_string_with_escape() {
let parser = parse_string_literal("\"hello\\nworld\"");
assert!(!parser.had_error);
let expected: Vec<u8> = b"hello\nworld".to_vec();
assert_eq!(parser.string_bytes, expected);
}
#[test]
fn test_parse_string_with_hex_escape() {
let parser = parse_string_literal("\"\\x48\\x65\\x6C\\x6C\\x6F\"");
assert!(!parser.had_error);
assert_eq!(parser.string_bytes, b"Hello".to_vec());
}
#[test]
fn test_parse_wide_string() {
let parser = parse_string_literal("L\"hello\"");
assert!(!parser.had_error);
assert_eq!(parser.prefix, StringPrefixKind::Wide);
}
#[test]
fn test_parse_utf8_string() {
let parser = parse_string_literal("u8\"hello\"");
assert!(!parser.had_error);
assert_eq!(parser.prefix, StringPrefixKind::Utf8);
}
#[test]
fn test_parse_utf16_string() {
let parser = parse_string_literal("u\"hello\"");
assert!(!parser.had_error);
assert_eq!(parser.prefix, StringPrefixKind::Utf16);
}
#[test]
fn test_parse_utf32_string() {
let parser = parse_string_literal("U\"hello\"");
assert!(!parser.had_error);
assert_eq!(parser.prefix, StringPrefixKind::Utf32);
}
#[test]
fn test_parse_raw_string() {
let parser = parse_string_literal("R\"(hello world)\"");
assert!(!parser.had_error);
assert!(parser.is_raw);
assert_eq!(parser.string_bytes, b"hello world".to_vec());
assert!(parser.is_complete);
}
#[test]
fn test_parse_raw_string_with_delimiter() {
let parser = parse_string_literal("R\"xyz(hello (world))xyz\"");
assert!(!parser.had_error);
assert!(parser.is_raw);
assert_eq!(parser.raw_delimiter, "xyz");
assert_eq!(parser.string_bytes, b"hello (world)".to_vec());
}
#[test]
fn test_parse_raw_wide_string() {
let parser = parse_string_literal("LR\"(hello)\"");
assert!(!parser.had_error);
assert!(parser.is_raw);
assert_eq!(parser.prefix, StringPrefixKind::Wide);
}
#[test]
fn test_parse_ucn_short_valid() {
let parser = parse_ucn("\\u03A9");
assert!(!parser.had_error);
assert!(parser.is_short_form);
assert_eq!(parser.codepoint, Some(0x03A9));
assert_eq!(parser.consumed, 6);
}
#[test]
fn test_parse_ucn_long_valid() {
let parser = parse_ucn("\\U0001F600");
assert!(!parser.had_error);
assert!(!parser.is_short_form);
assert_eq!(parser.codepoint, Some(0x1F600));
assert_eq!(parser.consumed, 10);
}
#[test]
fn test_parse_ucn_short_surrogate_invalid() {
let parser = parse_ucn("\\uD800");
assert!(parser.had_error);
}
#[test]
fn test_parse_ucn_long_surrogate_invalid() {
let parser = parse_ucn("\\U0000D800");
assert!(parser.had_error);
}
#[test]
fn test_parse_ucn_short_out_of_range() {
let parser = parse_ucn("\\uFFFF");
assert!(!parser.had_error);
}
#[test]
fn test_parse_ucn_basic_source_char_invalid_for_identifier() {
assert!(!is_valid_ucn_for_identifier(0x0041));
}
#[test]
fn test_parse_ucn_c1_control_invalid() {
assert!(!is_valid_ucn_for_identifier(0x0080));
}
#[test]
fn test_trigraph_replacement_number_sign() {
let result = replace_trigraphs("??=define FOO 1");
assert_eq!(result, "#define FOO 1");
}
#[test]
fn test_trigraph_replacement_left_bracket() {
let result = replace_trigraphs("arr??(0??)");
assert_eq!(result, "arr[0]");
}
#[test]
fn test_trigraph_replacement_backslash() {
let result = replace_trigraphs("abc??/ndef");
assert!(result.contains('\\'));
}
#[test]
fn test_trigraph_count() {
let source = "??= ??( ??/ ??) ??' ??< ??! ??> ??-";
assert_eq!(count_trigraphs(source), 9);
}
#[test]
fn test_trigraph_not_trigraph() {
let result = replace_trigraphs("?hello??world");
assert_eq!(result, "?hello??world");
}
#[test]
fn test_is_trigraph_number_sign() {
let result = is_trigraph("??=something");
assert_eq!(result, Some(('#', 3)));
}
#[test]
fn test_is_trigraph_none() {
assert_eq!(is_trigraph("?abc"), None);
assert_eq!(is_trigraph("abc"), None);
}
#[test]
fn test_digraph_left_bracket() {
let result = is_digraph("<:something");
assert_eq!(result, Some((TokenKind::LBracket, 2)));
}
#[test]
fn test_digraph_right_bracket() {
let result = is_digraph(":>");
assert_eq!(result, Some((TokenKind::RBracket, 2)));
}
#[test]
fn test_digraph_left_brace() {
let result = is_digraph("<%");
assert_eq!(result, Some((TokenKind::LBrace, 2)));
}
#[test]
fn test_digraph_right_brace() {
let result = is_digraph("%>");
assert_eq!(result, Some((TokenKind::RBrace, 2)));
}
#[test]
fn test_digraph_hash() {
let result = is_digraph("%:");
assert_eq!(result, Some((TokenKind::Hash, 2)));
}
#[test]
fn test_digraph_hashhash() {
let result = is_digraph("%:%:");
assert_eq!(result, Some((TokenKind::HashHash, 4)));
}
#[test]
fn test_digraph_hashhash_detection() {
assert!(is_digraph_hashhash("%:%:"));
assert!(!is_digraph_hashhash("%:"));
}
#[test]
fn test_digraph_count() {
let source = "<: :> <% %> %: %:%:";
assert_eq!(count_digraphs(source), 6);
}
#[test]
fn test_int_suffix_table_lookup_none() {
let entry = lookup_int_suffix("");
assert!(entry.is_some());
assert_eq!(entry.unwrap().kind, IntSuffixKind::None);
}
#[test]
fn test_int_suffix_table_lookup_u() {
let entry = lookup_int_suffix("u");
assert!(entry.is_some());
assert_eq!(entry.unwrap().kind, IntSuffixKind::Unsigned);
}
#[test]
fn test_int_suffix_table_lookup_ll() {
let entry = lookup_int_suffix("ll");
assert!(entry.is_some());
assert_eq!(entry.unwrap().kind, IntSuffixKind::LongLong);
}
#[test]
fn test_int_suffix_table_lookup_ull() {
let entry = lookup_int_suffix("ull");
assert!(entry.is_some());
assert_eq!(entry.unwrap().kind, IntSuffixKind::UnsignedLongLong);
}
#[test]
fn test_int_suffix_table_lookup_i64() {
let entry = lookup_int_suffix("i64");
assert!(entry.is_some());
assert_eq!(entry.unwrap().kind, IntSuffixKind::Int64);
}
#[test]
fn test_float_suffix_table_lookup_f() {
let entry = lookup_float_suffix("f");
assert!(entry.is_some());
assert_eq!(entry.unwrap().kind, FloatSuffixKind::Float);
}
#[test]
fn test_float_suffix_table_lookup_l() {
let entry = lookup_float_suffix("l");
assert!(entry.is_some());
assert_eq!(entry.unwrap().kind, FloatSuffixKind::LongDouble);
}
#[test]
fn test_float_suffix_table_lookup_f32() {
let entry = lookup_float_suffix("f32");
assert!(entry.is_some());
assert_eq!(entry.unwrap().kind, FloatSuffixKind::Float32);
}
#[test]
fn test_all_int_suffix_spellings_not_empty() {
let all = all_int_suffix_spellings();
assert!(!all.is_empty());
assert!(all.contains(&"u"));
assert!(all.contains(&"ll"));
assert!(all.contains(&"ull"));
}
#[test]
fn test_all_float_suffix_spellings_not_empty() {
let all = all_float_suffix_spellings();
assert!(!all.is_empty());
assert!(all.contains(&"f"));
assert!(all.contains(&"l"));
}
#[test]
fn test_try_lex_operator_plus() {
let result = try_lex_operator("+something");
assert_eq!(result, Some((TokenKind::Plus, 1)));
}
#[test]
fn test_try_lex_operator_plus_equal() {
let result = try_lex_operator("+= 5");
assert_eq!(result, Some((TokenKind::PlusEqual, 2)));
}
#[test]
fn test_try_lex_operator_plus_plus() {
let result = try_lex_operator("++i");
assert_eq!(result, Some((TokenKind::PlusPlus, 2)));
}
#[test]
fn test_try_lex_operator_less_less_equal() {
let result = try_lex_operator("<<= x");
assert_eq!(result, Some((TokenKind::LessLessEqual, 3)));
}
#[test]
fn test_try_lex_operator_spaceship() {
let result = try_lex_operator("<=>");
assert_eq!(result, Some((TokenKind::Spaceship, 3)));
}
#[test]
fn test_try_lex_operator_arrow_star() {
let result = try_lex_operator("->*");
assert_eq!(result, Some((TokenKind::ArrowStar, 3)));
}
#[test]
fn test_try_lex_operator_scope_resolution() {
let result = try_lex_operator("::value");
assert_eq!(result, Some((TokenKind::ScopeResolution, 2)));
}
#[test]
fn test_try_lex_operator_hashhash() {
let result = try_lex_operator("##");
assert_eq!(result, Some((TokenKind::HashHash, 2)));
}
#[test]
fn test_try_lex_operator_hash() {
let result = try_lex_operator("#define");
assert_eq!(result, Some((TokenKind::Hash, 1)));
}
#[test]
fn test_is_assignment_operator() {
assert!(is_assignment_operator(TokenKind::Equal));
assert!(is_assignment_operator(TokenKind::PlusEqual));
assert!(is_assignment_operator(TokenKind::StarEqual));
assert!(!is_assignment_operator(TokenKind::Plus));
assert!(!is_assignment_operator(TokenKind::Star));
}
#[test]
fn test_is_comparison_operator() {
assert!(is_comparison_operator(TokenKind::EqualEqual));
assert!(is_comparison_operator(TokenKind::NotEqual));
assert!(is_comparison_operator(TokenKind::Less));
assert!(is_comparison_operator(TokenKind::Spaceship));
assert!(!is_comparison_operator(TokenKind::Plus));
}
#[test]
fn test_is_preprocessor_token() {
assert!(is_preprocessor_token(TokenKind::Hash));
assert!(is_preprocessor_token(TokenKind::HashHash));
assert!(!is_preprocessor_token(TokenKind::Plus));
}
#[test]
fn test_is_logical_operator() {
assert!(is_logical_operator(TokenKind::AndAnd));
assert!(is_logical_operator(TokenKind::OrOr));
assert!(is_logical_operator(TokenKind::Exclaim));
assert!(!is_logical_operator(TokenKind::Ampersand));
}
#[test]
fn test_is_bitwise_operator() {
assert!(is_bitwise_operator(TokenKind::Ampersand));
assert!(is_bitwise_operator(TokenKind::Pipe));
assert!(is_bitwise_operator(TokenKind::Caret));
assert!(is_bitwise_operator(TokenKind::Tilde));
assert!(is_bitwise_operator(TokenKind::LessLess));
assert!(!is_bitwise_operator(TokenKind::AndAnd));
}
#[test]
fn test_operator_precedence() {
let mul_prec = operator_precedence(TokenKind::Star);
let add_prec = operator_precedence(TokenKind::Plus);
assert!(mul_prec < add_prec); }
#[test]
fn test_pp_directive_classification() {
assert_eq!(PPDirectiveKind::from_str("define"), PPDirectiveKind::Define);
assert_eq!(
PPDirectiveKind::from_str("include"),
PPDirectiveKind::Include
);
assert_eq!(PPDirectiveKind::from_str("ifdef"), PPDirectiveKind::Ifdef);
assert_eq!(PPDirectiveKind::from_str("endif"), PPDirectiveKind::Endif);
assert!(PPDirectiveKind::from_str("if").is_conditional_start());
assert!(PPDirectiveKind::from_str("if").is_conditional());
assert!(!PPDirectiveKind::from_str("endif").is_conditional());
}
#[test]
fn test_cpp_alternative_token_lookup() {
let result = lookup_cpp_alternative("and");
assert_eq!(result, Some(TokenKind::AndAnd));
let result = lookup_cpp_alternative("or");
assert_eq!(result, Some(TokenKind::OrOr));
let result = lookup_cpp_alternative("not");
assert_eq!(result, Some(TokenKind::Exclaim));
let result = lookup_cpp_alternative("xor");
assert_eq!(result, Some(TokenKind::Caret));
}
#[test]
fn test_cpp_alternative_token_not_found() {
let result = lookup_cpp_alternative("nand");
assert_eq!(result, None);
}
#[test]
fn test_lookup_named_character_null() {
assert_eq!(lookup_named_character("NULL"), Some(0x0000));
}
#[test]
fn test_lookup_named_character_newline() {
assert_eq!(lookup_named_character("LINE FEED"), Some(0x000A));
}
#[test]
fn test_lookup_named_character_euro() {
assert_eq!(lookup_named_character("EURO SIGN"), Some(0x20AC));
}
#[test]
fn test_lookup_named_character_case_insensitive() {
assert_eq!(lookup_named_character("null"), Some(0x0000));
}
#[test]
fn test_lookup_named_character_unknown() {
assert_eq!(lookup_named_character("NONEXISTENT CHARACTER"), None);
}
#[test]
fn test_parse_empty_numeric_literal() {
let result = parse_numeric_literal("", CLangStandard::C17);
assert!(result.had_error);
}
#[test]
fn test_parse_hex_float_large_exponent() {
let result = parse_numeric_literal("0x1.0p+1023", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.exponent, Some(1023));
}
#[test]
fn test_parse_hex_float_no_decimal_point() {
let result = parse_numeric_literal("0x1p10", CLangStandard::C17);
assert!(!result.had_error);
assert!(result.is_floating());
}
#[test]
fn test_parse_numeric_literal_multiple_errors() {
let mut parser = NumericLiteralParser::new("0b123", CLangStandard::C23);
parser.parse();
assert!(parser.result.had_error);
}
#[test]
fn test_parse_exponent_no_sign() {
let result = parse_numeric_literal("1e10", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.exponent, Some(10));
assert!(!result.exponent_is_negative);
}
#[test]
fn test_parse_exponent_explicit_positive() {
let result = parse_numeric_literal("1e+5", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.exponent, Some(5));
}
#[test]
fn test_parse_exponent_overflow() {
let result = parse_numeric_literal("1e99999999999999999999", CLangStandard::C17);
assert!(result.had_error);
}
#[test]
fn test_parse_unsigned_long_ul_spelling() {
let result = parse_numeric_literal("42UL", CLangStandard::C99);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::UnsignedLong);
}
#[test]
fn test_i64_on_float_error() {
let result = parse_numeric_literal("3.14i64", CLangStandard::C17);
assert!(result.had_error);
}
#[test]
fn test_parse_hex_mixed_case() {
let result = parse_numeric_literal("0xDeadBeef", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.significand, "DeadBeef");
}
#[test]
fn test_parse_binary_upper_prefix_with_suffix() {
let result = parse_numeric_literal("0B1010ULL", CLangStandard::C23);
assert!(!result.had_error);
assert_eq!(result.radix, NumericRadix::Binary);
assert_eq!(result.int_suffix, IntSuffixKind::UnsignedLongLong);
}
#[test]
fn test_zero_is_decimal_not_octal() {
let result = parse_numeric_literal("0", CLangStandard::C17);
assert_eq!(result.radix, NumericRadix::Decimal);
}
#[test]
fn test_octal_with_illegal_digits() {
let result = parse_numeric_literal("07", CLangStandard::C17);
assert!(!result.had_error);
}
#[test]
fn test_parse_float_imaginary() {
let result = parse_numeric_literal("1.0i", CLangStandard::C17);
assert!(!result.had_error);
assert!(result.is_floating());
assert_eq!(result.float_suffix, FloatSuffixKind::Imaginary);
}
#[test]
fn test_parse_float_imaginary_fi() {
let result = parse_numeric_literal("1.0fi", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.float_suffix, FloatSuffixKind::FloatImaginary);
}
#[test]
fn test_parse_integer_imaginary() {
let result = parse_numeric_literal("42i", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.int_suffix, IntSuffixKind::Imaginary);
}
#[test]
fn test_float_suffix_f32_is_c23() {
let result = parse_numeric_literal("1.0f32", CLangStandard::C23);
assert!(result.float_suffix.is_c23());
}
#[test]
fn test_bf16_suffix_is_c23() {
let result = parse_numeric_literal("1.0bf16", CLangStandard::C23);
assert!(result.float_suffix.is_c23());
}
#[test]
fn test_parse_c23_octal_prefix() {
let result = parse_numeric_literal("0o777", CLangStandard::C23);
assert!(!result.had_error);
assert_eq!(result.radix, NumericRadix::Octal);
}
#[test]
fn test_decimal_float_leading_zeros() {
let result = parse_numeric_literal("001.500", CLangStandard::C17);
assert!(!result.had_error);
assert!(result.is_floating());
}
#[test]
fn test_very_long_decimal_literal() {
let long_num = "1".repeat(1000);
let result = parse_numeric_literal(&long_num, CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.significand.len(), 1000);
}
#[test]
fn test_digit_separator_at_start_invalid() {
let result = parse_numeric_literal("0x'FF", CLangStandard::C23);
assert!(!result.had_digit_separators || result.had_error);
}
#[test]
fn test_digit_separator_at_end_invalid() {
let result = parse_numeric_literal("0xFF'", CLangStandard::C23);
assert!(!result.had_digit_separators || result.had_error);
}
#[test]
fn test_consecutive_digit_separators() {
let result = parse_numeric_literal("0xFF''FF", CLangStandard::C23);
assert!(!result.had_digit_separators || result.had_error);
}
#[test]
fn test_all_simple_escape_sequences() {
let escapes = [
("'\\a'", 0x07),
("'\\b'", 0x08),
("'\\f'", 0x0C),
("'\\n'", 0x0A),
("'\\r'", 0x0D),
("'\\t'", 0x09),
("'\\v'", 0x0B),
("'\\\\'", 0x5C),
("'\\''", 0x27),
("'\\\"'", 0x22),
("'\\?'", 0x3F),
];
for (input, expected) in &escapes {
let parser = parse_char_literal(input);
assert!(!parser.had_error, "failed on: {}", input);
assert_eq!(parser.value, *expected, "mismatch on: {}", input);
}
}
#[test]
fn test_empty_char_literal_error() {
let parser = parse_char_literal("''");
assert!(parser.had_error);
}
#[test]
fn test_unterminated_char_literal() {
let parser = parse_char_literal("'a");
assert!(parser.had_error);
assert!(!parser.is_complete);
}
#[test]
fn test_invalid_ucn_in_char_literal() {
let parser = parse_char_literal("'\\uD800'");
assert!(parser.had_error);
}
#[test]
fn test_truncated_ucn_in_char() {
let parser = parse_char_literal("'\\u03A'");
assert!(parser.had_error);
}
#[test]
fn test_gnu_escape_char() {
let parser = parse_char_literal("'\\e'");
assert!(!parser.had_error);
assert_eq!(parser.value, 0x1B);
}
#[test]
fn test_newline_in_char_literal_error() {
let parser = parse_char_literal("'\na'");
assert!(parser.had_error);
}
#[test]
fn test_string_with_null_terminator() {
let parser = parse_string_literal("\"hello\\0world\"");
assert!(!parser.had_error);
assert_eq!(parser.string_bytes.len(), 11); }
#[test]
fn test_string_with_ucn() {
let parser = parse_string_literal("\"\\u03A9\"");
assert!(!parser.had_error);
assert!(parser.string_codepoints.contains(&0x03A9));
}
#[test]
fn test_unterminated_string_literal() {
let parser = parse_string_literal("\"hello");
assert!(parser.had_error);
assert!(!parser.is_complete);
}
#[test]
fn test_raw_string_empty_delimiter() {
let parser = parse_string_literal("R\"(hello)\"");
assert!(!parser.had_error);
assert_eq!(parser.raw_delimiter, "");
assert_eq!(parser.string_bytes, b"hello".to_vec());
}
#[test]
fn test_raw_string_nested_parens() {
let parser = parse_string_literal("R\"x(hello (nested (world)))x\"");
assert!(!parser.had_error);
assert_eq!(parser.raw_delimiter, "x");
assert_eq!(parser.string_bytes, b"hello (nested (world))".to_vec());
}
#[test]
fn test_raw_string_unicode_content() {
let parser = parse_string_literal("R\"(café résumé)\"");
assert!(!parser.had_error);
}
#[test]
fn test_raw_utf8_string() {
let parser = parse_string_literal("u8R\"(hello)\"");
assert!(!parser.had_error);
assert!(parser.is_raw);
assert_eq!(parser.prefix, StringPrefixKind::Utf8);
}
#[test]
fn test_raw_utf16_string() {
let parser = parse_string_literal("uR\"(hello)\"");
assert!(!parser.had_error);
assert!(parser.is_raw);
assert_eq!(parser.prefix, StringPrefixKind::Utf16);
}
#[test]
fn test_raw_utf32_string() {
let parser = parse_string_literal("UR\"(hello)\"");
assert!(!parser.had_error);
assert!(parser.is_raw);
assert_eq!(parser.prefix, StringPrefixKind::Utf32);
}
#[test]
fn test_string_concat_prefix_mismatch() {
let result = parse_and_concatenate_strings(&["\"hello\"", "L\"world\""]);
assert!(result.had_error);
}
#[test]
fn test_ucn_zero_valid() {
let parser = parse_ucn("\\u0000");
assert!(!parser.had_error);
assert_eq!(parser.codepoint, Some(0));
}
#[test]
fn test_ucn_max_valid() {
let parser = parse_ucn("\\U0010FFFF");
assert!(!parser.had_error);
assert_eq!(parser.codepoint, Some(0x10FFFF));
}
#[test]
fn test_ucn_beyond_max_invalid() {
let parser = parse_ucn("\\U00110000");
assert!(parser.had_error);
}
#[test]
fn test_ucn_dollar_allowed() {
assert!(is_valid_ucn_for_identifier(0x0024)); }
#[test]
fn test_ucn_at_allowed() {
assert!(is_valid_ucn_for_identifier(0x0040)); }
#[test]
fn test_ucn_backtick_allowed() {
assert!(is_valid_ucn_for_identifier(0x0060)); }
#[test]
fn test_ucn_del_invalid() {
assert!(!is_valid_ucn_for_identifier(0x007F));
}
#[test]
fn test_validate_ucn_for_identifier_valid() {
let validity = validate_ucn(0x03A9, true, true);
assert_eq!(validity, UcnValidity::Valid);
}
#[test]
fn test_validate_ucn_literal_only() {
let validity = validate_ucn(0x0041, true, true); assert_eq!(validity, UcnValidity::Invalid);
}
#[test]
fn test_validate_ucn_in_literal() {
let validity = validate_ucn(0x0041, true, false); assert_eq!(validity, UcnValidity::Valid);
}
#[test]
fn test_parse_ucn_for_identifier_rejects_basic() {
let parser = parse_ucn_for_identifier("\\u0041"); assert!(parser.had_error);
}
#[test]
fn test_parse_ucn_for_identifier_accepts_greek() {
let parser = parse_ucn_for_identifier("\\u03B1"); assert!(!parser.had_error);
}
#[test]
fn test_trigraph_all_replacements() {
for entry in &TRIGRAPH_TABLE {
let result = replace_trigraphs(entry.sequence);
assert_eq!(
result,
entry.replacement.to_string(),
"trigraph {} should become {}",
entry.sequence,
entry.replacement
);
}
}
#[test]
fn test_digraph_hashhash_length() {
let entry = lookup_digraph("%:%:").unwrap();
assert_eq!(entry.length, 4);
}
#[test]
fn test_digraph_hash_length() {
let entry = lookup_digraph("%:").unwrap();
assert_eq!(entry.length, 2);
}
#[test]
fn test_is_digraph_none() {
assert_eq!(is_digraph("abc"), None);
assert_eq!(is_digraph(":<"), None); }
#[test]
fn test_lookup_operator_by_spelling() {
let op = lookup_operator_by_spelling("<<=").unwrap();
assert_eq!(op.kind, TokenKind::LessLessEqual);
assert!(op.is_assignment);
assert_eq!(op.length, 3);
}
#[test]
fn test_operator_longest_match() {
let result = try_lex_operator("<<=x");
assert_eq!(result, Some((TokenKind::LessLessEqual, 3)));
}
#[test]
fn test_operator_less_less() {
let result = try_lex_operator("<<x");
assert_eq!(result, Some((TokenKind::LessLess, 2)));
}
#[test]
fn test_all_cpp_alternatives() {
for entry in CPP_ALTERNATIVE_TOKENS.iter() {
let result = lookup_cpp_alternative(entry.keyword);
assert_eq!(result, Some(entry.kind));
}
}
#[test]
fn test_pp_directive_is_conditional() {
assert!(PPDirectiveKind::If.is_conditional());
assert!(PPDirectiveKind::Ifdef.is_conditional());
assert!(PPDirectiveKind::Ifndef.is_conditional());
assert!(PPDirectiveKind::Elif.is_conditional());
assert!(PPDirectiveKind::Elifdef.is_conditional());
assert!(PPDirectiveKind::Elifndef.is_conditional());
assert!(!PPDirectiveKind::Else.is_conditional());
assert!(!PPDirectiveKind::Endif.is_conditional());
assert!(!PPDirectiveKind::Define.is_conditional());
}
#[test]
fn test_pp_directive_requires_expression() {
assert!(PPDirectiveKind::If.requires_expression());
assert!(PPDirectiveKind::Elif.requires_expression());
assert!(!PPDirectiveKind::Ifdef.requires_expression());
assert!(!PPDirectiveKind::Define.requires_expression());
assert!(!PPDirectiveKind::Else.requires_expression());
}
#[test]
fn test_classify_pp_token() {
assert_eq!(
classify_pp_token(TokenKind::PPDefine),
Some(PPDirectiveKind::Define)
);
assert_eq!(
classify_pp_token(TokenKind::PPIf),
Some(PPDirectiveKind::If)
);
assert_eq!(classify_pp_token(TokenKind::KwIf), None);
}
#[test]
fn test_split_suffix_parts_complex() {
assert_eq!(split_suffix_parts("ull"), vec!["u", "ll"]);
assert_eq!(split_suffix_parts("llu"), vec!["ll", "u"]);
assert_eq!(split_suffix_parts("uz"), vec!["u", "z"]);
assert_eq!(split_suffix_parts("zwb"), vec!["z", "wb"]);
}
#[test]
fn test_numeric_radix_prefix() {
assert_eq!(NumericRadix::Binary.prefix(), Some("0b"));
assert_eq!(NumericRadix::Hex.prefix(), Some("0x"));
assert_eq!(NumericRadix::Octal.prefix(), None);
assert_eq!(NumericRadix::Decimal.prefix(), None);
}
#[test]
fn test_numeric_radix_display() {
assert_eq!(format!("{}", NumericRadix::Binary), "binary");
assert_eq!(format!("{}", NumericRadix::Octal), "octal");
assert_eq!(format!("{}", NumericRadix::Decimal), "decimal");
assert_eq!(format!("{}", NumericRadix::Hex), "hexadecimal");
}
#[test]
fn test_char_prefix_kind_properties() {
assert_eq!(CharPrefixKind::Narrow.prefix_str(), "");
assert_eq!(CharPrefixKind::Wide.prefix_str(), "L");
assert_eq!(CharPrefixKind::Utf8.prefix_str(), "u8");
assert_eq!(CharPrefixKind::Utf16.prefix_str(), "u");
assert_eq!(CharPrefixKind::Utf32.prefix_str(), "U");
assert_eq!(CharPrefixKind::Narrow.char_width(), 1);
assert_eq!(CharPrefixKind::Utf16.char_width(), 2);
assert_eq!(CharPrefixKind::Wide.char_width(), 4);
assert!(CharPrefixKind::Utf8.is_unicode());
assert!(CharPrefixKind::Utf16.is_unicode());
assert!(CharPrefixKind::Utf32.is_unicode());
assert!(!CharPrefixKind::Narrow.is_unicode());
assert!(!CharPrefixKind::Wide.is_unicode());
}
#[test]
fn test_string_prefix_kind_properties() {
assert_eq!(StringPrefixKind::Narrow.char_width(), 1);
assert_eq!(StringPrefixKind::Utf16.char_width(), 2);
assert_eq!(StringPrefixKind::Wide.char_width(), 4);
}
#[test]
fn test_escape_value_codepoint() {
assert_eq!(EscapeValue::Literal('A').codepoint(), 0x41);
assert_eq!(EscapeValue::Alert.codepoint(), 0x07);
assert_eq!(EscapeValue::Newline.codepoint(), 0x0A);
assert_eq!(EscapeValue::Octal(0o77).codepoint(), 0o77);
assert_eq!(EscapeValue::Hex(0x41).codepoint(), 0x41);
assert_eq!(EscapeValue::UcnShort(0x03A9).codepoint(), 0x03A9);
assert_eq!(EscapeValue::UcnLong(0x1F600).codepoint(), 0x1F600);
}
#[test]
fn test_escape_value_is_escaped() {
assert!(!EscapeValue::Literal('a').is_escaped());
assert!(EscapeValue::Newline.is_escaped());
assert!(EscapeValue::Hex(0x41).is_escaped());
}
#[test]
fn test_numeric_result_is_unsigned() {
let mut result = NumericLiteralResult::new(NumericRadix::Decimal);
result.int_suffix = IntSuffixKind::Unsigned;
assert!(result.is_unsigned());
result.int_suffix = IntSuffixKind::None;
assert!(!result.is_unsigned());
}
#[test]
fn test_int_suffix_kind_name() {
assert_eq!(IntSuffixKind::None.name(), "int");
assert_eq!(IntSuffixKind::UnsignedLong.name(), "unsigned long int");
assert_eq!(IntSuffixKind::BitInt(32).name(), "_BitInt(N)");
}
#[test]
fn test_float_suffix_is_decimal() {
assert!(FloatSuffixKind::Decimal32.is_decimal());
assert!(FloatSuffixKind::Decimal64.is_decimal());
assert!(FloatSuffixKind::Decimal128.is_decimal());
assert!(!FloatSuffixKind::Float.is_decimal());
}
#[test]
fn test_float_suffix_is_imaginary() {
assert!(FloatSuffixKind::Imaginary.is_imaginary());
assert!(FloatSuffixKind::FloatImaginary.is_imaginary());
assert!(!FloatSuffixKind::Float.is_imaginary());
}
#[test]
fn test_radix_base_values() {
assert_eq!(NumericRadix::Binary.base(), 2);
assert_eq!(NumericRadix::Octal.base(), 8);
assert_eq!(NumericRadix::Decimal.base(), 10);
assert_eq!(NumericRadix::Hex.base(), 16);
}
#[test]
fn test_radix_valid_digit() {
assert!(NumericRadix::Binary.is_valid_digit('0'));
assert!(NumericRadix::Binary.is_valid_digit('1'));
assert!(!NumericRadix::Binary.is_valid_digit('2'));
assert!(NumericRadix::Octal.is_valid_digit('7'));
assert!(!NumericRadix::Octal.is_valid_digit('8'));
assert!(NumericRadix::Decimal.is_valid_digit('9'));
assert!(!NumericRadix::Decimal.is_valid_digit('a'));
assert!(NumericRadix::Hex.is_valid_digit('F'));
assert!(NumericRadix::Hex.is_valid_digit('a'));
assert!(!NumericRadix::Hex.is_valid_digit('G'));
}
#[test]
fn test_radix_digit_value() {
assert_eq!(NumericRadix::Hex.digit_value('A'), Some(10));
assert_eq!(NumericRadix::Hex.digit_value('F'), Some(15));
assert_eq!(NumericRadix::Hex.digit_value('5'), Some(5));
assert_eq!(NumericRadix::Hex.digit_value('G'), None);
}
#[test]
fn test_int_suffix_kind_is_unsigned() {
assert!(!IntSuffixKind::None.is_unsigned());
assert!(IntSuffixKind::Unsigned.is_unsigned());
assert!(IntSuffixKind::UnsignedLong.is_unsigned());
assert!(IntSuffixKind::UnsignedLongLong.is_unsigned());
assert!(!IntSuffixKind::Long.is_unsigned());
}
#[test]
fn test_float_suffix_kind_names() {
assert_eq!(FloatSuffixKind::None.name(), "double");
assert_eq!(FloatSuffixKind::Float.name(), "float");
assert_eq!(FloatSuffixKind::LongDouble.name(), "long double");
assert_eq!(FloatSuffixKind::Float16.name(), "_Float16");
}
#[test]
fn test_string_concatenation_two_segments() {
let result = parse_and_concatenate_strings(&["\"hello\"", "\" world\""]);
assert!(!result.had_error);
assert_eq!(result.num_concatenated, 2);
assert_eq!(result.string_bytes, b"hello world".to_vec());
}
#[test]
fn test_string_concatenation_three_segments() {
let result = parse_and_concatenate_strings(&["\"a\"", "\"b\"", "\"c\""]);
assert!(!result.had_error);
assert_eq!(result.num_concatenated, 3);
assert_eq!(result.string_bytes, b"abc".to_vec());
}
#[test]
fn test_string_concatenation_empty_input() {
let result = parse_and_concatenate_strings(&[]);
assert!(result.had_error);
}
#[test]
fn test_is_ident_start() {
assert!(is_ident_start('a'));
assert!(is_ident_start('Z'));
assert!(is_ident_start('_'));
assert!(!is_ident_start('0'));
assert!(!is_ident_start(' '));
}
#[test]
fn test_is_ident_continue() {
assert!(is_ident_continue('a'));
assert!(is_ident_continue('0'));
assert!(is_ident_continue('_'));
assert!(!is_ident_continue(' '));
}
#[test]
fn test_is_whitespace() {
assert!(is_whitespace(' '));
assert!(is_whitespace('\t'));
assert!(is_whitespace('\n'));
assert!(is_whitespace('\r'));
assert!(!is_whitespace('a'));
}
#[test]
fn test_hex_value_table() {
assert_eq!(hex_value('0'), 0);
assert_eq!(hex_value('9'), 9);
assert_eq!(hex_value('A'), 10);
assert_eq!(hex_value('a'), 10);
assert_eq!(hex_value('F'), 15);
assert_eq!(hex_value('f'), 15);
}
#[test]
fn test_split_suffix_parts_u() {
assert_eq!(split_suffix_parts("u"), vec!["u"]);
}
#[test]
fn test_split_suffix_parts_ull() {
assert_eq!(split_suffix_parts("ull"), vec!["u", "ll"]);
}
#[test]
fn test_split_suffix_parts_lu() {
assert_eq!(split_suffix_parts("lu"), vec!["l", "u"]);
}
#[test]
fn test_int_suffix_table_no_duplicate_canonical_suffixes() {
let mut seen = std::collections::HashSet::new();
for entry in INT_SUFFIX_TABLE.iter() {
assert!(
seen.insert(entry.suffix),
"duplicate canonical suffix: {}",
entry.suffix
);
}
}
#[test]
fn test_float_suffix_table_no_duplicate_canonical_suffixes() {
let mut seen = std::collections::HashSet::new();
for entry in FLOAT_SUFFIX_TABLE.iter() {
assert!(
seen.insert(entry.suffix),
"duplicate canonical suffix: {}",
entry.suffix
);
}
}
#[test]
fn test_int_suffix_c23_marking() {
assert!(IntSuffixKind::Size.is_c23());
assert!(IntSuffixKind::UnsignedSize.is_c23());
assert!(IntSuffixKind::BitPrecise.is_c23());
assert!(!IntSuffixKind::Long.is_c23());
assert!(!IntSuffixKind::UnsignedLongLong.is_c23());
}
#[test]
fn test_operator_table_sorted_descending_length() {
for window in OPERATOR_TABLE.windows(2) {
assert!(
window[0].length >= window[1].length,
"OPERATOR_TABLE not sorted by descending length: {} (len {}) before {} (len {})",
window[0].spelling,
window[0].length,
window[1].spelling,
window[1].length,
);
}
}
#[test]
fn test_stress_parse_numeric_literals() {
let literals = [
"0", "1", "42", "0xFF", "0b1010", "0777", "3.14", "1e10", "0x1.0p0", "1u", "2L", "3LL",
"4uLL", "5.0f", "6.0L", "7i64", "8u64", "9i128", "10u128", "11z", "12uz", "13wb",
"14f16", "15f32", "16f64", "17f128", "18bf16",
];
for lit in &literals {
let result = parse_numeric_literal(lit, CLangStandard::C23);
assert!(!result.had_error, "failed to parse: {}", lit);
}
}
#[test]
fn test_stress_parse_char_literals() {
let literals = [
"'a'",
"'\\n'",
"'\\t'",
"'\\0'",
"'\\077'",
"'\\x41'",
"L'a'",
"u'a'",
"U'a'",
"u8'a'",
"'\\u03A9'",
"L'\\u03A9'",
"'\\u0041'",
];
for lit in &literals {
let parser = parse_char_literal(lit);
if parser.had_error {
eprintln!("Errors for {}: {:?}", lit, parser.errors);
}
assert!(!parser.had_error, "failed to parse char literal: {}", lit);
}
}
#[test]
fn test_stress_parse_string_literals() {
let literals = [
"\"hello\"",
"L\"hello\"",
"u8\"hello\"",
"u\"hello\"",
"U\"hello\"",
"\"hello\\nworld\"",
"R\"(hello)\"",
"LR\"(hello)\"",
];
for lit in &literals {
let parser = parse_string_literal(lit);
if parser.had_error {
eprintln!("Errors for {}: {:?}", lit, parser.errors);
}
assert!(!parser.had_error, "failed to parse string: {}", lit);
}
}
#[test]
fn test_classify_token_keyword() {
assert_eq!(classify_token(TokenKind::KwInt), TokenCategory::Keyword);
assert_eq!(classify_token(TokenKind::KwIf), TokenCategory::Keyword);
assert_eq!(classify_token(TokenKind::KwReturn), TokenCategory::Keyword);
}
#[test]
fn test_classify_token_literals() {
assert_eq!(
classify_token(TokenKind::NumericLiteral),
TokenCategory::NumericLiteral
);
assert_eq!(
classify_token(TokenKind::CharLiteral),
TokenCategory::CharLiteral
);
assert_eq!(
classify_token(TokenKind::StringLiteral),
TokenCategory::StringLiteral
);
assert_eq!(
classify_token(TokenKind::WideCharLiteral),
TokenCategory::CharLiteral
);
}
#[test]
fn test_classify_token_operators() {
assert_eq!(classify_token(TokenKind::Plus), TokenCategory::Operator);
assert_eq!(
classify_token(TokenKind::EqualEqual),
TokenCategory::Operator
);
assert_eq!(classify_token(TokenKind::LParen), TokenCategory::Operator);
}
#[test]
fn test_classify_token_preprocessor() {
assert_eq!(
classify_token(TokenKind::PPDefine),
TokenCategory::Preprocessor
);
assert_eq!(
classify_token(TokenKind::PPInclude),
TokenCategory::Preprocessor
);
assert_eq!(classify_token(TokenKind::Hash), TokenCategory::Preprocessor);
}
#[test]
fn test_classify_token_eof() {
assert_eq!(classify_token(TokenKind::Eof), TokenCategory::Eof);
}
#[test]
fn test_reconstruct_decimal_integer() {
let result = parse_numeric_literal("42", CLangStandard::C17);
let reconstructed = reconstruct_numeric_literal(&result);
assert_eq!(reconstructed, "42");
}
#[test]
fn test_reconstruct_hex_with_suffix() {
let result = parse_numeric_literal("0xDEADull", CLangStandard::C99);
let reconstructed = reconstruct_numeric_literal(&result);
assert_eq!(reconstructed, "0xDEADull");
}
#[test]
fn test_reconstruct_float() {
let result = parse_numeric_literal("3.14f", CLangStandard::C17);
let reconstructed = reconstruct_numeric_literal(&result);
assert_eq!(reconstructed, "3.14f");
}
#[test]
fn test_reconstruct_hex_float() {
let result = parse_numeric_literal("0x1.0p+10", CLangStandard::C17);
let reconstructed = reconstruct_numeric_literal(&result);
assert_eq!(reconstructed, "0x1.0p+10");
}
#[test]
fn test_reconstruct_binary() {
let result = parse_numeric_literal("0b1010", CLangStandard::C23);
let reconstructed = reconstruct_numeric_literal(&result);
assert_eq!(reconstructed, "0b1010");
}
#[test]
fn test_reconstruct_narrow_char() {
let parser = parse_char_literal("'a'");
let reconstructed = reconstruct_char_literal(&parser);
assert_eq!(reconstructed, "'a'");
}
#[test]
fn test_reconstruct_char_escape() {
let parser = parse_char_literal("'\\n'");
let reconstructed = reconstruct_char_literal(&parser);
assert_eq!(reconstructed, "'\\n'");
}
#[test]
fn test_reconstruct_wide_char() {
let parser = parse_char_literal("L'a'");
let reconstructed = reconstruct_char_literal(&parser);
assert_eq!(reconstructed, "L'a'");
}
#[test]
fn test_needs_escape_in_string() {
assert!(needs_escape_in_string('"'));
assert!(needs_escape_in_string('\\'));
assert!(needs_escape_in_string('\n'));
assert!(needs_escape_in_string('\0'));
assert!(!needs_escape_in_string('a'));
}
#[test]
fn test_escape_for_string() {
assert_eq!(escape_for_string('"'), "\\\"");
assert_eq!(escape_for_string('\\'), "\\\\");
assert_eq!(escape_for_string('\n'), "\\n");
assert_eq!(escape_for_string('a'), "a");
}
#[test]
fn test_cfeature_long_long_available_in_c99() {
assert!(CFeature::LongLong.is_available_in(CLangStandard::C99));
assert!(!CFeature::LongLong.is_available_in(CLangStandard::C89));
}
#[test]
fn test_cfeature_binary_in_c23() {
assert!(CFeature::BinaryLiterals.is_available_in(CLangStandard::C23));
assert!(!CFeature::BinaryLiterals.is_available_in(CLangStandard::C17));
}
#[test]
fn test_cfeature_digit_separators_c23_only() {
assert!(CFeature::DigitSeparators.is_available_in(CLangStandard::C23));
assert!(!CFeature::DigitSeparators.is_available_in(CLangStandard::C17));
}
#[test]
fn test_cfeature_trigraphs_not_in_c23() {
assert!(CFeature::Trigraphs.is_available_in(CLangStandard::C17));
assert!(!CFeature::Trigraphs.is_available_in(CLangStandard::C23));
}
#[test]
fn test_cfeature_fixed_width_in_gnu() {
assert!(CFeature::FixedWidthSuffixes.is_available_in(CLangStandard::Gnu11));
assert!(CFeature::ImaginaryLiterals.is_available_in(CLangStandard::Gnu99));
}
#[test]
fn test_features_for_standard_not_empty() {
let features = features_for_standard(CLangStandard::C23);
assert!(!features.is_empty());
assert!(features.len() >= 10);
}
#[test]
fn test_c89_minimal_features() {
let features = features_for_standard(CLangStandard::C89);
assert!(!features.contains(&CFeature::LongLong));
assert!(features.contains(&CFeature::Digraphs));
}
#[test]
fn test_deep_lexer_default() {
let lexer = DeepLexer::default();
assert_eq!(lexer.standard, CLangStandard::C17);
}
#[test]
fn test_deep_lexer_numeric() {
let lexer = DeepLexer::new(CLangStandard::C17);
let result = lexer.lex_numeric_literal("0xFF");
assert_eq!(result.radix, NumericRadix::Hex);
}
#[test]
fn test_deep_lexer_char() {
let lexer = DeepLexer::new(CLangStandard::C17);
let parser = lexer.lex_char_literal("'x'");
assert_eq!(parser.value, 'x' as u32);
}
#[test]
fn test_deep_lexer_string() {
let lexer = DeepLexer::new(CLangStandard::C17);
let parser = lexer.lex_string_literal("\"test\"");
assert_eq!(parser.string_bytes, b"test".to_vec());
}
#[test]
fn test_deep_lexer_has_feature() {
let lexer = DeepLexer::new(CLangStandard::C23);
assert!(lexer.has_feature(CFeature::BinaryLiterals));
assert!(!lexer.has_feature(CFeature::Trigraphs));
}
#[test]
fn test_deep_lexer_operator() {
let lexer = DeepLexer::new(CLangStandard::C17);
let result = lexer.lex_operator("+=");
assert_eq!(result, Some((TokenKind::PlusEqual, 2)));
}
#[test]
fn test_deep_lexer_digraph() {
let lexer = DeepLexer::new(CLangStandard::C17);
let result = lexer.lex_digraph("<:");
assert_eq!(result, Some((TokenKind::LBracket, 2)));
}
#[test]
fn test_deep_lexer_trigraph_replace() {
let lexer = DeepLexer::new(CLangStandard::C11);
let result = lexer.replace_trigraphs("??=define");
assert_eq!(result, "#define");
}
#[test]
fn test_deep_lexer_classify() {
let lexer = DeepLexer::new(CLangStandard::C17);
assert_eq!(lexer.classify(TokenKind::KwInt), TokenCategory::Keyword);
assert_eq!(
lexer.classify(TokenKind::NumericLiteral),
TokenCategory::NumericLiteral
);
}
#[test]
fn test_reconstruct_size_suffix() {
let result = parse_numeric_literal("100z", CLangStandard::C23);
let reconstructed = reconstruct_numeric_literal(&result);
assert_eq!(reconstructed, "100z");
}
#[test]
fn test_reconstruct_bit_precise() {
let result = parse_numeric_literal("42wb", CLangStandard::C23);
let reconstructed = reconstruct_numeric_literal(&result);
assert_eq!(reconstructed, "42wb");
}
#[test]
fn test_reconstruct_i64() {
let result = parse_numeric_literal("42i64", CLangStandard::C17);
let reconstructed = reconstruct_numeric_literal(&result);
assert_eq!(reconstructed, "42i64");
}
#[test]
fn test_all_string_prefixes() {
let cases = [
("\"hi\"", StringPrefixKind::Narrow),
("L\"hi\"", StringPrefixKind::Wide),
("u8\"hi\"", StringPrefixKind::Utf8),
("u\"hi\"", StringPrefixKind::Utf16),
("U\"hi\"", StringPrefixKind::Utf32),
];
for (input, expected_prefix) in &cases {
let parser = parse_string_literal(input);
assert_eq!(parser.prefix, *expected_prefix, "failed on: {}", input);
}
}
#[test]
fn test_all_char_prefixes() {
let cases = [
("'a'", CharPrefixKind::Narrow),
("L'a'", CharPrefixKind::Wide),
("u8'a'", CharPrefixKind::Utf8),
("u'a'", CharPrefixKind::Utf16),
("U'a'", CharPrefixKind::Utf32),
];
for (input, expected_prefix) in &cases {
let parser = parse_char_literal(input);
assert_eq!(parser.prefix, *expected_prefix, "failed on: {}", input);
}
}
#[test]
fn test_zero_in_all_radices() {
let cases = [
("0", NumericRadix::Decimal),
("0x0", NumericRadix::Hex),
("0b0", NumericRadix::Binary),
];
for (input, expected_radix) in &cases {
let result = parse_numeric_literal(input, CLangStandard::C23);
assert_eq!(result.radix, *expected_radix, "failed on: {}", input);
assert_eq!(result.significand, "0");
}
}
#[test]
fn test_hex_float_binary_exponent() {
let result = parse_numeric_literal("0x1.0p10", CLangStandard::C17);
assert!(!result.had_error);
assert!(result.exponent.is_some());
}
#[test]
fn test_octal_recognition() {
let result = parse_numeric_literal("0777", CLangStandard::C17);
assert_eq!(result.radix, NumericRadix::Octal);
}
#[test]
fn test_leading_zero_but_decimal() {
let result = parse_numeric_literal("08", CLangStandard::C17);
}
#[test]
fn test_zero_dot_five_is_float() {
let result = parse_numeric_literal("0.5", CLangStandard::C17);
assert!(result.is_floating());
assert_eq!(result.radix, NumericRadix::Decimal);
}
#[test]
fn test_hex_float_negative_exponent() {
let result = parse_numeric_literal("0x1.0p-10", CLangStandard::C17);
assert!(!result.had_error);
assert_eq!(result.exponent, Some(-10));
}
#[test]
fn test_three_char_multi_char_literal() {
let parser = parse_char_literal("'abc'");
assert!(!parser.had_error);
assert!(parser.is_multi_char);
assert_eq!(parser.multi_char_values.len(), 3);
}
#[test]
fn test_string_with_multiple_escapes() {
let parser = parse_string_literal("\"\\t\\n\\r\"");
assert!(!parser.had_error);
assert_eq!(parser.string_bytes, vec![0x09, 0x0A, 0x0D]);
}
#[test]
fn test_string_embedded_ucn() {
let parser = parse_string_literal("\"hello\\u03A9world\"");
assert!(!parser.had_error);
assert!(parser.string_codepoints.contains(&0x03A9));
}
#[test]
fn test_is_digit_all() {
for ch in '0'..='9' {
assert!(is_digit(ch), "{} should be a digit", ch);
}
assert!(!is_digit('a'));
assert!(!is_digit('.'));
}
#[test]
fn test_is_hex_digit_all() {
for ch in '0'..='9' {
assert!(is_hex_digit(ch));
}
for ch in 'a'..='f' {
assert!(is_hex_digit(ch));
}
for ch in 'A'..='F' {
assert!(is_hex_digit(ch));
}
assert!(!is_hex_digit('g'));
assert!(!is_hex_digit('G'));
}
#[test]
fn test_is_octal_digit_all() {
for ch in '0'..='7' {
assert!(is_octal_digit(ch));
}
assert!(!is_octal_digit('8'));
assert!(!is_octal_digit('9'));
}
#[test]
fn test_is_binary_digit() {
assert!(is_binary_digit('0'));
assert!(is_binary_digit('1'));
assert!(!is_binary_digit('2'));
}
#[test]
fn test_is_horizontal_whitespace() {
assert!(is_horizontal_whitespace(' '));
assert!(is_horizontal_whitespace('\t'));
assert!(!is_horizontal_whitespace('\n'));
assert!(!is_horizontal_whitespace('a'));
}
#[test]
fn test_ucn_short_form_bmp_limit() {
assert!(is_valid_ucn_value(0xFFFF, true));
assert!(!is_valid_ucn_value(0x10000, true));
}
#[test]
fn test_validate_ucn_all_scenarios() {
assert_eq!(validate_ucn(0x03A9, true, true), UcnValidity::Valid);
assert_eq!(validate_ucn(0x03A9, true, false), UcnValidity::Valid);
assert_eq!(validate_ucn(0xD800, true, false), UcnValidity::Invalid);
assert_eq!(validate_ucn(0x110000, false, false), UcnValidity::Invalid);
assert_eq!(validate_ucn(0x10000, true, false), UcnValidity::Invalid);
}
#[test]
fn test_int_suffix_default() {
assert_eq!(IntSuffixKind::default(), IntSuffixKind::None);
}
#[test]
fn test_float_suffix_default() {
assert_eq!(FloatSuffixKind::default(), FloatSuffixKind::None);
}
#[test]
fn test_numeric_result_defaults() {
let result = NumericLiteralResult::new(NumericRadix::Hex);
assert_eq!(result.radix, NumericRadix::Hex);
assert!(result.significand.is_empty());
assert_eq!(result.exponent, None);
assert!(!result.has_decimal_point);
assert_eq!(result.kind, NumericLiteralKind::Integer);
assert!(!result.had_error);
}
#[test]
fn test_roundtrip_numeric_literals() {
let inputs = [
"42", "0xFF", "0b1010", "3.14", "1.5e10", "0x1.0p0", "1u", "2L", "3LL", "4uLL", "5.0f",
"6.0L",
];
for input in &inputs {
let result = parse_numeric_literal(input, CLangStandard::C23);
let reconstructed = reconstruct_numeric_literal(&result);
assert!(!result.had_error, "parse error on: {}", input);
}
}
#[test]
fn test_roundtrip_char_literals() {
let inputs = ["'a'", "'\\n'", "L'a'", "u'a'", "U'a'"];
for input in &inputs {
let parser = parse_char_literal(input);
assert!(!parser.had_error, "parse error on: {}", input);
let reconstructed = reconstruct_char_literal(&parser);
assert!(!reconstructed.is_empty());
}
}
#[test]
fn test_operator_precedence_non_operator() {
assert_eq!(operator_precedence(TokenKind::Eof), 0);
assert_eq!(operator_precedence(TokenKind::Identifier), 0);
assert_eq!(operator_precedence(TokenKind::NumericLiteral), 0);
}
#[test]
fn test_try_lex_operator_none() {
assert_eq!(try_lex_operator("abc"), None);
assert_eq!(try_lex_operator("123"), None);
assert_eq!(try_lex_operator("_foo"), None);
}
#[test]
fn test_operator_table_entries_valid() {
for entry in OPERATOR_TABLE.iter() {
let result = try_lex_operator(entry.spelling);
assert_eq!(
result,
Some((entry.kind, entry.length)),
"operator table entry '{}' should lex correctly",
entry.spelling
);
}
}
#[test]
fn test_digraph_table_entries_valid() {
for entry in DIGRAPH_TABLE.iter() {
let result = is_digraph(entry.sequence);
assert_eq!(
result,
Some((entry.token_kind, entry.length)),
"digraph table entry '{}' should match",
entry.sequence
);
}
}
#[test]
fn test_trigraph_table_entries_valid() {
for entry in TRIGRAPH_TABLE.iter() {
let result = is_trigraph(entry.sequence);
assert_eq!(
result,
Some((entry.replacement, 3)),
"trigraph '{}' should match",
entry.sequence
);
}
}
#[test]
fn test_operator_precedence_ordering() {
let postfix = operator_precedence(TokenKind::PlusPlus); let prefix = operator_precedence(TokenKind::Exclaim); let mul = operator_precedence(TokenKind::Star); let add = operator_precedence(TokenKind::Plus); let shift = operator_precedence(TokenKind::LessLess); let rel = operator_precedence(TokenKind::Less); let eq = operator_precedence(TokenKind::EqualEqual); let bit_and = operator_precedence(TokenKind::Ampersand); assert!(mul < add);
assert!(add < shift);
assert!(shift < rel);
assert!(rel < eq);
}
#[test]
fn test_reconstruct_hex_float_negative_exp() {
let result = parse_numeric_literal("0x1.0p-3", CLangStandard::C17);
let reconstructed = reconstruct_numeric_literal(&result);
assert_eq!(reconstructed, "0x1.0p-3");
}
#[test]
fn test_reconstruct_decimal_float_negative_exp() {
let result = parse_numeric_literal("1.5e-3", CLangStandard::C17);
let reconstructed = reconstruct_numeric_literal(&result);
assert_eq!(reconstructed, "1.5e-3");
}
#[test]
fn test_token_kind_imports() {
let _ = Token {
kind: TokenKind::Eof,
text: String::new(),
location: SourceLoc::unknown(),
flags: TokenFlags::none(),
};
}
#[test]
fn test_clang_standard_integration() {
let std = CLangStandard::C23;
assert_eq!(std.as_str(), "c23");
assert!(std.has_long_long());
assert!(std.has_bool());
assert!(!std.is_gnu());
}
#[test]
fn test_type_availability() {
let _parser = NumericLiteralParser::new("42", CLangStandard::C17);
let _char_parser = CharLiteralParser::new("'a'");
let _str_parser = StringLiteralParser::new("\"hi\"");
let _ucn_parser = UcnParser::new("\\u03A9");
let _lexer = DeepLexer::new(CLangStandard::C17);
}
}