use crate::latin_1::{char_to_latin1, Latin1Str, Latin1String, NonLatin1CharError};
use crate::tokens::{Keyword, Token, TokenKind, Trivia, TriviaPiece};
fn default_trivia() -> Trivia {
Trivia::from([TriviaPiece::Spaces(1)])
}
macro_rules! domain_type {
($(#[$meta:meta])* struct $name:ident($token_kind:expr);) => {
$(#[$meta])*
pub struct $name(Token);
impl $name {
pub fn with_trivia(mut self, trivia: Trivia) -> Self {
self.0.set_leading_trivia(trivia);
self
}
}
impl From<$name> for Token {
fn from(value: $name) -> Token {
value.0
}
}
impl From<Token> for $name {
fn from(token: Token) -> Self {
assert_eq!(token.kind(), $token_kind);
$name(token)
}
}
};
}
domain_type!(
struct Identifier(TokenKind::Identifier);
);
impl Identifier {
pub fn new(value: Latin1String) -> Self {
Identifier(Token::new(TokenKind::Identifier, value, default_trivia()))
}
}
impl From<&[u8]> for Identifier {
fn from(value: &[u8]) -> Self {
Identifier::new(Latin1String::from(value))
}
}
impl<const N: usize> From<[u8; N]> for Identifier {
fn from(value: [u8; N]) -> Self {
Identifier::new(Latin1String::from(&value))
}
}
impl<const N: usize> From<&[u8; N]> for Identifier {
fn from(value: &[u8; N]) -> Self {
Identifier::new(Latin1String::from(value))
}
}
impl TryFrom<&str> for Identifier {
type Error = crate::latin_1::Utf8ToLatin1Error;
fn try_from(value: &str) -> Result<Self, Self::Error> {
Ok(Identifier::new(Latin1String::from_utf8(value)?))
}
}
impl TryFrom<String> for Identifier {
type Error = crate::latin_1::Utf8ToLatin1Error;
fn try_from(value: String) -> Result<Self, Self::Error> {
Ok(Identifier::new(Latin1String::from_utf8(&value)?))
}
}
#[test]
fn from_token_works_for_correctly_kinded_tokens() {
let tok = Token::new(TokenKind::Identifier, b"foo", Trivia::default());
let id = Identifier::from(tok);
let out: Token = id.into();
assert_eq!(out.text(), "foo");
}
#[test]
#[should_panic(expected = "Identifier")]
fn from_token_panics_with_wong_kind() {
let tok = Token::new(
TokenKind::AbstractLiteral,
b"42".as_slice(),
Trivia::default(),
);
let _ = Identifier::from(tok);
}
domain_type!(
struct AbstractLiteral(TokenKind::AbstractLiteral);
);
impl AbstractLiteral {
fn new(text: impl AsRef<Latin1Str>) -> Self {
AbstractLiteral(Token::new(
TokenKind::AbstractLiteral,
text,
default_trivia(),
))
}
pub fn integer(value: u64) -> Self {
Self::new(Latin1Str::new(format!("{}", value).as_bytes()))
}
pub fn real(value: f64) -> Self {
let s = format!("{}", value);
let text = if s.contains('.') || s.contains('e') || s.contains('E') {
s
} else {
format!("{}.0", s)
};
Self::new(Latin1Str::new(text.as_bytes()))
}
}
#[test]
fn abstract_literal_real_always_has_decimal_point() {
let tok: Token = AbstractLiteral::real(1.0).into();
assert_eq!(tok.kind(), TokenKind::AbstractLiteral);
assert_eq!(tok.text().to_string(), "1.0");
let tok: Token = AbstractLiteral::real(2.5).into();
assert_eq!(tok.text().to_string(), "2.5");
let tok: Token = AbstractLiteral::real(1e10).into();
let text = tok.text().to_string();
assert!(
text.contains('.') || text.contains('e') || text.contains('E'),
"expected decimal point or exponent in {text}"
);
}
#[test]
fn abstract_literal_integer_produces_plain_decimal_string() {
let tok: Token = AbstractLiteral::integer(42).into();
assert_eq!(tok.kind(), TokenKind::AbstractLiteral);
assert_eq!(tok.text().to_string(), "42");
}
domain_type!(
struct StringLiteral(TokenKind::StringLiteral);
);
impl StringLiteral {
pub fn new(content: &str) -> Self {
let mut encoded = Latin1String::with_capacity(content.len() + 2);
encoded.push(b'"');
for byte in content.bytes() {
if byte == b'"' {
encoded.push(b'"');
encoded.push(b'"');
} else {
encoded.push(byte);
}
}
encoded.push(b'"');
StringLiteral(Token::new(
TokenKind::StringLiteral,
encoded,
default_trivia(),
))
}
}
#[test]
fn string_literal_adds_surrounding_quotes_and_doubles_embedded_quotes() {
let tok: Token = StringLiteral::new("say \"hi\"").into();
assert_eq!(tok.kind(), TokenKind::StringLiteral);
assert_eq!(tok.text().to_string(), "\"say \"\"hi\"\"\"");
}
domain_type!(
struct CharLiteral(TokenKind::CharacterLiteral);
);
impl TryFrom<char> for CharLiteral {
type Error = NonLatin1CharError;
fn try_from(value: char) -> Result<Self, Self::Error> {
Ok(CharLiteral::new(char_to_latin1(value)?))
}
}
impl CharLiteral {
pub fn new(byte: u8) -> Self {
let text = [b'\'', byte, b'\''];
CharLiteral(Token::new(
TokenKind::CharacterLiteral,
text,
default_trivia(),
))
}
}
#[test]
fn char_literal_has_tick_marks() {
let tok: Token = CharLiteral::new(b'A').into();
assert_eq!(tok.kind(), TokenKind::CharacterLiteral);
assert_eq!(tok.text().to_string(), "'A'");
}
#[test]
fn char_literal_produces_correct_output_with_quote_char_itself() {
let tok: Token = CharLiteral::new(b'\'').into();
assert_eq!(tok.text().to_string(), "'''");
}
#[test]
fn char_literal_from_char_rejects_non_latin_1() {
assert!(CharLiteral::try_from('€').is_err());
assert!(CharLiteral::try_from('A').is_ok());
}
domain_type!(
struct BitStringLiteral(TokenKind::BitStringLiteral);
);
impl BitStringLiteral {
fn new(prefix: &Latin1Str, digits: &Latin1Str) -> Self {
let mut text = prefix.to_latin1_string();
text.push(b'"');
text.extend(digits);
text.push(b'"');
BitStringLiteral(Token::new(
TokenKind::BitStringLiteral,
text,
default_trivia(),
))
}
pub fn binary(digits: impl AsRef<Latin1Str>) -> Self {
Self::new(Latin1Str::new(b"B"), digits.as_ref())
}
pub fn binary_unsigned(digits: impl AsRef<Latin1Str>) -> Self {
Self::new(Latin1Str::new(b"UB"), digits.as_ref())
}
pub fn binary_signed(digits: impl AsRef<Latin1Str>) -> Self {
Self::new(Latin1Str::new(b"SB"), digits.as_ref())
}
pub fn octal(digits: impl AsRef<Latin1Str>) -> Self {
Self::new(Latin1Str::new(b"O"), digits.as_ref())
}
pub fn octal_unsigned(digits: impl AsRef<Latin1Str>) -> Self {
Self::new(Latin1Str::new(b"UO"), digits.as_ref())
}
pub fn octal_signed(digits: impl AsRef<Latin1Str>) -> Self {
Self::new(Latin1Str::new(b"SO"), digits.as_ref())
}
pub fn hex(digits: impl AsRef<Latin1Str>) -> Self {
Self::new(Latin1Str::new(b"X"), digits.as_ref())
}
pub fn hex_unsigned(digits: impl AsRef<Latin1Str>) -> Self {
Self::new(Latin1Str::new(b"UX"), digits.as_ref())
}
pub fn hex_signed(digits: impl AsRef<Latin1Str>) -> Self {
Self::new(Latin1Str::new(b"SX"), digits.as_ref())
}
pub fn decimal(digits: impl AsRef<Latin1Str>) -> Self {
Self::new(Latin1Str::new(b"D"), digits.as_ref())
}
}
#[test]
fn bit_string_literal_helpers_produce_correct_prefixes_prefixes() {
let tok: Token = BitStringLiteral::binary(b"1010").into();
assert_eq!(tok.text().to_string(), r#"B"1010""#);
let tok: Token = BitStringLiteral::octal(b"77").into();
assert_eq!(tok.text().to_string(), r#"O"77""#);
let tok: Token = BitStringLiteral::hex(b"FF").into();
assert_eq!(tok.text().to_string(), r#"X"FF""#);
}
impl TokenKind {
pub(crate) fn canonical_token(&self) -> Option<Token> {
self.canonical_text()
.map(|text| Token::new(*self, text, default_trivia()))
}
}
impl Keyword {
pub(crate) fn canonical_token(&self) -> Token {
Token::new(
TokenKind::Keyword(*self),
self.canonical_text(),
default_trivia(),
)
}
}