use crate::frontend::ast::{Expr, ExprKind, Span};
use crate::frontend::lexer::Token;
use crate::frontend::parser::{bail, ParserState, Result};
pub(in crate::frontend::parser) fn parse_identifier_token(
state: &mut ParserState,
token: &Token,
span: Span,
) -> Result<Expr> {
match token {
Token::Identifier(name) => {
state.tokens.advance();
if !state.in_guard_context && matches!(state.tokens.peek(), Some((Token::FatArrow, _)))
{
let ident_expr = Expr::new(ExprKind::Identifier(name.clone()), span);
super::super::parse_lambda_from_expr(state, ident_expr, span)
} else {
Ok(Expr::new(ExprKind::Identifier(name.clone()), span))
}
}
Token::Underscore => {
state.tokens.advance();
Ok(Expr::new(ExprKind::Identifier("_".to_string()), span))
}
Token::Self_ => {
state.tokens.advance();
Ok(Expr::new(ExprKind::Identifier("self".to_string()), span))
}
Token::Super => {
state.tokens.advance();
Ok(Expr::new(ExprKind::Identifier("super".to_string()), span))
}
_ => bail!("Expected identifier token, got: {token:?}"),
}
}
pub(in crate::frontend::parser) fn parse_module_path_segments(
state: &mut ParserState,
initial: String,
) -> Result<String> {
let mut path = vec![initial];
while matches!(state.tokens.peek(), Some((Token::ColonColon, _))) {
state.tokens.advance(); path.push(parse_path_segment(state)?);
}
Ok(path.join("::"))
}
pub(in crate::frontend::parser) fn parse_path_segment(state: &mut ParserState) -> Result<String> {
if matches!(state.tokens.peek(), Some((Token::Less, _))) {
parse_turbofish_generics(state)
} else if let Some((Token::Identifier(segment), _)) = state.tokens.peek() {
let segment = segment.clone();
state.tokens.advance();
Ok(segment)
} else if matches!(state.tokens.peek(), Some((Token::Star, _))) {
state.tokens.advance();
Ok("*".to_string())
} else if let Some((token, _)) = state.tokens.peek() {
let name = token_to_keyword_string(token);
if name.is_empty() {
bail!("Expected identifier or '*' after '::'")
}
state.tokens.advance();
Ok(name)
} else {
bail!("Expected identifier or '*' after '::'")
}
}
pub(in crate::frontend::parser) fn parse_turbofish_generics(
state: &mut ParserState,
) -> Result<String> {
state.tokens.advance();
let mut type_args = Vec::new();
loop {
let type_str = parse_turbofish_type(state)?;
type_args.push(type_str);
match state.tokens.peek() {
Some((Token::Comma, _)) => {
state.tokens.advance(); }
Some((Token::Greater, _)) => {
state.tokens.advance(); break;
}
Some((Token::GreaterEqual, _)) => {
state.tokens.advance();
break;
}
_ => bail!("Expected ',' or '>' in turbofish generics"),
}
}
Ok(format!("<{}>", type_args.join(", ")))
}
fn parse_turbofish_type(state: &mut ParserState) -> Result<String> {
let mut type_str = String::new();
loop {
match state.tokens.peek() {
Some((Token::Identifier(name), _)) => {
type_str.push_str(name);
state.tokens.advance();
if matches!(state.tokens.peek(), Some((Token::ColonColon, _))) {
type_str.push_str("::");
state.tokens.advance();
continue;
}
break;
}
Some((Token::Integer(n), _)) => {
type_str.push_str(&n.clone());
state.tokens.advance();
break;
}
_ => break,
}
}
if matches!(state.tokens.peek(), Some((Token::Less, _))) {
let nested = parse_turbofish_generics(state)?;
type_str.push_str(&nested);
}
Ok(type_str)
}
pub(in crate::frontend::parser) fn token_to_keyword_string(token: &Token) -> String {
match token {
Token::As => "as".to_string(),
Token::Async => "async".to_string(),
Token::Await => "await".to_string(),
Token::Break => "break".to_string(),
Token::Const => "const".to_string(),
Token::Continue => "continue".to_string(),
Token::Crate => "crate".to_string(),
Token::Default => "default".to_string(),
Token::Else => "else".to_string(),
Token::Enum => "enum".to_string(),
Token::Err => "Err".to_string(),
Token::Fn => "fn".to_string(),
Token::For => "for".to_string(),
Token::From => "from".to_string(),
Token::Fun => "fun".to_string(),
Token::If => "if".to_string(),
Token::Impl => "impl".to_string(),
Token::In => "in".to_string(),
Token::Let => "let".to_string(),
Token::Loop => "loop".to_string(),
Token::Match => "match".to_string(),
Token::Mod => "mod".to_string(),
Token::Module => "module".to_string(),
Token::Mut => "mut".to_string(),
Token::None => "None".to_string(),
Token::Ok => "Ok".to_string(),
Token::Private => "private".to_string(),
Token::Pub => "pub".to_string(),
Token::Return => "return".to_string(),
Token::Self_ => "self".to_string(),
Token::Some => "Some".to_string(),
Token::Static => "static".to_string(),
Token::Struct => "struct".to_string(),
Token::Super => "super".to_string(),
Token::Trait => "trait".to_string(),
Token::Type => "type".to_string(),
Token::Unsafe => "unsafe".to_string(),
Token::Use => "use".to_string(),
Token::Where => "where".to_string(),
Token::While => "while".to_string(),
Token::Option => "Option".to_string(),
Token::Result => "Result".to_string(),
_ => String::new(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::frontend::parser::Parser;
#[test]
fn test_simple_identifier() {
let code = "foo";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Simple identifier should parse");
}
#[test]
fn test_identifier_with_numbers() {
let code = "foo123";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Identifier with numbers should parse");
}
#[test]
fn test_identifier_with_underscores() {
let code = "foo_bar_baz";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Identifier with underscores should parse");
}
#[test]
fn test_identifier_starting_with_underscore() {
let code = "_foo";
let result = Parser::new(code).parse();
assert!(
result.is_ok(),
"Identifier starting with underscore should parse"
);
}
#[test]
fn test_underscore_identifier() {
let code = "_";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Underscore identifier should parse");
}
#[test]
fn test_self_identifier() {
let code = "self";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Self identifier should parse");
}
#[test]
fn test_super_identifier() {
let code = "super";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Super identifier should parse");
}
#[test]
fn test_default_identifier() {
let code = "default";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Default identifier should parse");
}
#[test]
fn test_qualified_path() {
let code = "math::add";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Qualified path should parse");
}
#[test]
fn test_nested_qualified_path() {
let code = "std::collections::HashMap";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Nested qualified path should parse");
}
#[test]
fn test_deeply_nested_qualified_path() {
let code = "a::b::c::d";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Deeply nested qualified path should parse");
}
#[test]
fn test_qualified_path_four_segments() {
let code = "mod1::mod2::mod3::item";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Four segment path should parse");
}
#[test]
fn test_qualified_path_with_super() {
let code = "super";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Super identifier should parse");
}
#[test]
fn test_qualified_path_with_self() {
let code = "self";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Self identifier should parse");
}
#[test]
fn test_generic_type_annotation() {
let code = "let x: Vec<i32> = Vec::new()";
let result = Parser::new(code).parse();
assert!(
result.is_ok(),
"Generic type annotation should parse: {result:?}"
);
}
#[test]
fn test_generic_function_call() {
let code = "collect::<Vec<_>>()";
let result = Parser::new(code).parse();
let _ = result; }
#[test]
fn test_nested_generic_types() {
let code = "let x: Vec<Vec<i32>> = vec![]";
let result = Parser::new(code).parse();
assert!(
result.is_ok(),
"Nested generic types should parse: {result:?}"
);
}
#[test]
fn test_multiple_type_params() {
let code = "let x: HashMap<String, i32> = HashMap::new()";
let result = Parser::new(code).parse();
assert!(
result.is_ok(),
"Multiple type params should parse: {result:?}"
);
}
#[test]
fn test_fat_arrow_lambda() {
let code = "x => x + 1";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Fat arrow lambda should parse");
}
#[test]
fn test_fat_arrow_lambda_multiplication() {
let code = "x => x * 2";
let result = Parser::new(code).parse();
assert!(
result.is_ok(),
"Fat arrow lambda with multiplication should parse"
);
}
#[test]
fn test_fat_arrow_lambda_complex_body() {
let code = "n => { let result = n * n; result }";
let result = Parser::new(code).parse();
assert!(
result.is_ok(),
"Fat arrow lambda with block body should parse"
);
}
#[test]
fn test_wildcard_use() {
let code = "use std::*";
let result = Parser::new(code).parse();
assert!(result.is_ok(), "Use with wildcard should parse: {result:?}");
}
#[test]
fn test_wildcard_in_nested_use() {
let code = "use std::collections::*";
let result = Parser::new(code).parse();
assert!(
result.is_ok(),
"Nested use with wildcard should parse: {result:?}"
);
}
#[test]
fn test_keyword_as_path_segment() {
let code = "crate::r#mod::item";
let result = Parser::new(code).parse();
assert!(result.is_ok() || result.is_err()); }
#[test]
fn test_token_to_keyword_string_as() {
assert_eq!(token_to_keyword_string(&Token::As), "as");
}
#[test]
fn test_token_to_keyword_string_async() {
assert_eq!(token_to_keyword_string(&Token::Async), "async");
}
#[test]
fn test_token_to_keyword_string_await() {
assert_eq!(token_to_keyword_string(&Token::Await), "await");
}
#[test]
fn test_token_to_keyword_string_break() {
assert_eq!(token_to_keyword_string(&Token::Break), "break");
}
#[test]
fn test_token_to_keyword_string_const() {
assert_eq!(token_to_keyword_string(&Token::Const), "const");
}
#[test]
fn test_token_to_keyword_string_continue() {
assert_eq!(token_to_keyword_string(&Token::Continue), "continue");
}
#[test]
fn test_token_to_keyword_string_crate() {
assert_eq!(token_to_keyword_string(&Token::Crate), "crate");
}
#[test]
fn test_token_to_keyword_string_default() {
assert_eq!(token_to_keyword_string(&Token::Default), "default");
}
#[test]
fn test_token_to_keyword_string_else() {
assert_eq!(token_to_keyword_string(&Token::Else), "else");
}
#[test]
fn test_token_to_keyword_string_enum() {
assert_eq!(token_to_keyword_string(&Token::Enum), "enum");
}
#[test]
fn test_token_to_keyword_string_err() {
assert_eq!(token_to_keyword_string(&Token::Err), "Err");
}
#[test]
fn test_token_to_keyword_string_fn() {
assert_eq!(token_to_keyword_string(&Token::Fn), "fn");
}
#[test]
fn test_token_to_keyword_string_for() {
assert_eq!(token_to_keyword_string(&Token::For), "for");
}
#[test]
fn test_token_to_keyword_string_from() {
assert_eq!(token_to_keyword_string(&Token::From), "from");
}
#[test]
fn test_token_to_keyword_string_fun() {
assert_eq!(token_to_keyword_string(&Token::Fun), "fun");
}
#[test]
fn test_token_to_keyword_string_if() {
assert_eq!(token_to_keyword_string(&Token::If), "if");
}
#[test]
fn test_token_to_keyword_string_impl() {
assert_eq!(token_to_keyword_string(&Token::Impl), "impl");
}
#[test]
fn test_token_to_keyword_string_in() {
assert_eq!(token_to_keyword_string(&Token::In), "in");
}
#[test]
fn test_token_to_keyword_string_let() {
assert_eq!(token_to_keyword_string(&Token::Let), "let");
}
#[test]
fn test_token_to_keyword_string_loop() {
assert_eq!(token_to_keyword_string(&Token::Loop), "loop");
}
#[test]
fn test_token_to_keyword_string_match() {
assert_eq!(token_to_keyword_string(&Token::Match), "match");
}
#[test]
fn test_token_to_keyword_string_mod() {
assert_eq!(token_to_keyword_string(&Token::Mod), "mod");
}
#[test]
fn test_token_to_keyword_string_module() {
assert_eq!(token_to_keyword_string(&Token::Module), "module");
}
#[test]
fn test_token_to_keyword_string_mut() {
assert_eq!(token_to_keyword_string(&Token::Mut), "mut");
}
#[test]
fn test_token_to_keyword_string_none() {
assert_eq!(token_to_keyword_string(&Token::None), "None");
}
#[test]
fn test_token_to_keyword_string_ok() {
assert_eq!(token_to_keyword_string(&Token::Ok), "Ok");
}
#[test]
fn test_token_to_keyword_string_private() {
assert_eq!(token_to_keyword_string(&Token::Private), "private");
}
#[test]
fn test_token_to_keyword_string_pub() {
assert_eq!(token_to_keyword_string(&Token::Pub), "pub");
}
#[test]
fn test_token_to_keyword_string_return() {
assert_eq!(token_to_keyword_string(&Token::Return), "return");
}
#[test]
fn test_token_to_keyword_string_self() {
assert_eq!(token_to_keyword_string(&Token::Self_), "self");
}
#[test]
fn test_token_to_keyword_string_some() {
assert_eq!(token_to_keyword_string(&Token::Some), "Some");
}
#[test]
fn test_token_to_keyword_string_static() {
assert_eq!(token_to_keyword_string(&Token::Static), "static");
}
#[test]
fn test_token_to_keyword_string_struct() {
assert_eq!(token_to_keyword_string(&Token::Struct), "struct");
}
#[test]
fn test_token_to_keyword_string_super() {
assert_eq!(token_to_keyword_string(&Token::Super), "super");
}
#[test]
fn test_token_to_keyword_string_trait() {
assert_eq!(token_to_keyword_string(&Token::Trait), "trait");
}
#[test]
fn test_token_to_keyword_string_type() {
assert_eq!(token_to_keyword_string(&Token::Type), "type");
}
#[test]
fn test_token_to_keyword_string_unsafe() {
assert_eq!(token_to_keyword_string(&Token::Unsafe), "unsafe");
}
#[test]
fn test_token_to_keyword_string_use() {
assert_eq!(token_to_keyword_string(&Token::Use), "use");
}
#[test]
fn test_token_to_keyword_string_where() {
assert_eq!(token_to_keyword_string(&Token::Where), "where");
}
#[test]
fn test_token_to_keyword_string_while() {
assert_eq!(token_to_keyword_string(&Token::While), "while");
}
#[test]
fn test_token_to_keyword_string_option() {
assert_eq!(token_to_keyword_string(&Token::Option), "Option");
}
#[test]
fn test_token_to_keyword_string_result() {
assert_eq!(token_to_keyword_string(&Token::Result), "Result");
}
#[test]
fn test_token_to_keyword_string_non_keyword() {
assert_eq!(token_to_keyword_string(&Token::Plus), "");
assert_eq!(token_to_keyword_string(&Token::Minus), "");
assert_eq!(token_to_keyword_string(&Token::Star), "");
}
use crate::frontend::ast::{Expr, ExprKind};
use crate::frontend::parser::Result;
fn parse(code: &str) -> Result<Expr> {
Parser::new(code).parse()
}
fn get_block_exprs(expr: &Expr) -> Option<&Vec<Expr>> {
match &expr.kind {
ExprKind::Block(exprs) => Some(exprs),
_ => None,
}
}
#[test]
fn test_identifier_produces_identifier_exprkind() {
let expr = parse("foo").unwrap();
if let Some(exprs) = get_block_exprs(&expr) {
assert!(matches!(&exprs[0].kind, ExprKind::Identifier(name) if name == "foo"));
}
}
#[test]
fn test_underscore_produces_identifier_exprkind() {
let expr = parse("_").unwrap();
if let Some(exprs) = get_block_exprs(&expr) {
assert!(matches!(&exprs[0].kind, ExprKind::Identifier(name) if name == "_"));
}
}
#[test]
fn test_self_produces_identifier_exprkind() {
let expr = parse("self").unwrap();
if let Some(exprs) = get_block_exprs(&expr) {
assert!(matches!(&exprs[0].kind, ExprKind::Identifier(name) if name == "self"));
}
}
#[test]
fn test_single_char_identifier() {
let result = parse("x");
assert!(result.is_ok());
}
#[test]
fn test_two_char_identifier() {
let result = parse("ab");
assert!(result.is_ok());
}
#[test]
fn test_long_identifier() {
let result = parse("very_long_identifier_name_here");
assert!(result.is_ok());
}
#[test]
fn test_identifier_all_underscores() {
let result = parse("___");
assert!(result.is_ok());
}
#[test]
fn test_identifier_mixed_case() {
let result = parse("camelCase");
assert!(result.is_ok());
}
#[test]
fn test_identifier_pascal_case() {
let result = parse("PascalCase");
assert!(result.is_ok());
}
#[test]
fn test_path_five_segments() {
let result = parse("a::b::c::d::e");
assert!(result.is_ok());
}
#[test]
fn test_path_six_segments() {
let result = parse("a::b::c::d::e::f");
assert!(result.is_ok());
}
#[test]
fn test_path_with_long_names() {
let result = parse("long_module::another_long::item");
assert!(result.is_ok());
}
#[test]
fn test_fat_arrow_simple() {
let result = parse("x => x");
assert!(result.is_ok());
}
#[test]
fn test_fat_arrow_with_block() {
let result = parse("x => { x + 1 }");
assert!(result.is_ok());
}
#[test]
fn test_fat_arrow_with_if() {
let result = parse("x => if x > 0 { x } else { 0 }");
assert!(result.is_ok());
}
#[test]
fn test_two_identifiers() {
let result = parse("a\nb");
assert!(result.is_ok());
}
#[test]
fn test_identifier_in_let() {
let result = parse("let x = y");
assert!(result.is_ok());
}
#[test]
fn test_identifier_in_function() {
let result = parse("fun f(x) { x }");
assert!(result.is_ok());
}
#[cfg(test)]
mod property_tests {
use super::*;
use proptest::prelude::*;
fn valid_identifier() -> impl Strategy<Value = String> {
prop::string::string_regex("[a-zA-Z_][a-zA-Z0-9_]*")
.unwrap()
.prop_filter("Must not be a keyword", |s| {
!matches!(
s.as_str(),
"fn" | "fun"
| "let"
| "var"
| "if"
| "else"
| "for"
| "while"
| "loop"
| "match"
| "break"
| "continue"
| "return"
| "async"
| "await"
| "try"
| "catch"
| "throw"
| "in"
| "as"
| "is"
| "self"
| "super"
| "mod"
| "use"
| "pub"
| "const"
| "static"
| "mut"
| "ref"
| "type"
| "struct"
| "enum"
| "trait"
| "impl"
)
})
}
proptest! {
#[test]
#[ignore = "Property tests run with --ignored flag"] fn prop_valid_identifiers_always_parse(ident in valid_identifier()) {
let code = ident.clone();
let result = Parser::new(&code).parse();
prop_assert!(result.is_ok(), "Valid identifier {} should parse", ident);
}
#[test]
#[ignore = "Property tests run with --ignored flag"]
fn prop_qualified_paths_parse(
mod1 in valid_identifier(),
mod2 in valid_identifier()
) {
let code = format!("{mod1}::{mod2}");
let result = Parser::new(&code).parse();
prop_assert!(result.is_ok(), "Qualified path {}::{} should parse", mod1, mod2);
}
#[test]
#[ignore = "Property tests run with --ignored flag"]
fn prop_triple_qualified_paths_parse(
mod1 in valid_identifier(),
mod2 in valid_identifier(),
mod3 in valid_identifier()
) {
let code = format!("{mod1}::{mod2}::{mod3}");
let result = Parser::new(&code).parse();
prop_assert!(result.is_ok(), "Triple path {}::{}::{} should parse", mod1, mod2, mod3);
}
#[test]
#[ignore = "Property tests run with --ignored flag"]
fn prop_keywords_as_path_segments(
keyword in prop::sample::select(vec![
"as", "for", "if", "match", "while", "let", "fn", "mod"
])
) {
let code = format!("crate::{keyword}");
let result = Parser::new(&code).parse();
prop_assert!(result.is_ok(), "Keyword {} as path segment should parse", keyword);
}
#[test]
#[ignore = "Property tests run with --ignored flag"]
fn prop_special_identifiers_always_parse(
special in prop::sample::select(vec!["_", "self", "super", "default"])
) {
let code = special.to_string();
let result = Parser::new(&code).parse();
prop_assert!(result.is_ok(), "Special identifier {} should parse", special);
}
#[test]
#[ignore = "Property tests run with --ignored flag"]
fn prop_fat_arrow_lambdas_parse(param in valid_identifier()) {
let code = format!("{param} => {param} + 1");
let result = Parser::new(&code).parse();
prop_assert!(result.is_ok(), "Fat arrow lambda with {} should parse", param);
}
}
}
}