use super::string::decode_char;
use super::*;
fn tok_types(input: &str) -> Vec<TokenType> {
tokenize(input).iter().map(|t| t.ty).collect()
}
fn tok_literals(input: &str) -> Vec<String> {
tokenize(input)
.iter()
.map(|t| t.literal.to_string())
.collect()
}
#[test]
fn test_decode_char_bounds_validation_to_its_sequence() {
assert_eq!(decode_char(&[0xC2, 0xA0, 0xFF]), Some(('\u{00A0}', 2)));
assert_eq!(
decode_char(&[0xF0, 0x9F, 0x98, 0x80, 0xFF]),
Some(('\u{1F600}', 4))
);
assert_eq!(decode_char(&[0xC2]), None);
let (c, len) = decode_char("\u{00E9}".as_bytes()).unwrap();
assert_eq!((c, len), ('\u{00E9}', 2));
assert!(!c.is_whitespace());
}
#[test]
fn test_unicode_whitespace_between_tokens() {
assert_eq!(
tok_types("RETURN\u{00A0}1"),
vec![TokenType::RETURN, TokenType::Integer]
);
assert_eq!(
tok_types("MATCH\u{2003}RETURN"),
vec![TokenType::MATCH, TokenType::RETURN]
);
}
#[test]
fn test_worst_case_unicode_whitespace_run_tokenizes() {
let mut q = String::from("RETURN");
q.extend(std::iter::repeat_n('\u{00A0}', 4096));
q.push('1');
assert_eq!(tok_types(&q), vec![TokenType::RETURN, TokenType::Integer]);
}
#[test]
fn test_multibyte_non_whitespace_is_not_skipped() {
let tokens = tokenize("\u{00A0}\u{00A7}");
assert!(!tokens.is_empty());
assert!(tokens.iter().all(|t| t.ty == TokenType::Error));
}
#[test]
fn test_invalid_multibyte_char_is_one_error_with_detail() {
for c in [
'\u{2014}', '\u{2013}', '\u{2212}', '\u{2010}', '\u{1F600}', '\u{00A7}', '\u{FEFF}', ] {
let owned = c.to_string();
let tokens = tokenize(&owned);
assert_eq!(tokens.len(), 1, "expected one token for {c:?}: {tokens:?}");
assert_eq!(tokens[0].ty, TokenType::Error, "{c:?}");
assert_eq!(
tokens[0].detail,
Some("InvalidUnicodeCharacter"),
"expected InvalidUnicodeCharacter for {c:?}, got {:?}",
tokens[0].detail
);
assert!(
tokens[0].literal.contains(c),
"literal must contain the decoded {c:?}: {:?}",
tokens[0].literal
);
}
}
#[test]
fn test_invalid_unicode_hyphen_lexes_following_integer() {
let tokens = tokenize("RETURN 42 \u{2014} 41");
let types: Vec<_> = tokens.iter().map(|t| t.ty).collect();
assert_eq!(
types,
vec![
TokenType::RETURN,
TokenType::Integer,
TokenType::Error,
TokenType::Integer,
]
);
let err = &tokens[2];
assert_eq!(err.detail, Some("InvalidUnicodeCharacter"));
assert!(err.literal.contains('\u{2014}'));
assert_eq!(err.col, 11);
assert_eq!(tokens[3].literal, "41");
assert_eq!(tokens[3].col, 15);
}
#[test]
fn test_ascii_fallback_keeps_unexpected_character_without_detail() {
for (query, ch) in [("RETURN 1 # 2", '#'), ("RETURN 1 ? 2", '?')] {
let tokens = tokenize(query);
let err = tokens
.iter()
.find(|t| t.ty == TokenType::Error)
.unwrap_or_else(|| panic!("expected an error token for {query:?}"));
assert_eq!(err.detail, None, "{query:?}");
assert_eq!(err.literal, format!("unexpected character: {ch}"));
}
}
#[test]
fn test_simple_punctuation() {
assert_eq!(
tok_types("()[]{}:,;|*+/%^="),
vec![
TokenType::LeftParen,
TokenType::RightParen,
TokenType::LeftBracket,
TokenType::RightBracket,
TokenType::LeftBrace,
TokenType::RightBrace,
TokenType::Colon,
TokenType::Comma,
TokenType::Semicolon,
TokenType::Pipe,
TokenType::Star,
TokenType::Plus,
TokenType::Slash,
TokenType::Percent,
TokenType::Caret,
TokenType::Equals,
]
);
}
#[test]
fn test_arrows_and_comparisons() {
assert_eq!(
tok_types("-> <- <> <= >= < > - ."),
vec![
TokenType::ArrowRight,
TokenType::ArrowLeft,
TokenType::NotEquals,
TokenType::LessEqual,
TokenType::GreaterEqual,
TokenType::LessThan,
TokenType::GreaterThan,
TokenType::Dash,
TokenType::Dot,
]
);
}
#[test]
fn test_dot_dot() {
assert_eq!(tok_types(".."), vec![TokenType::DotDot]);
assert_eq!(
tok_types("1..5"),
vec![TokenType::Integer, TokenType::DotDot, TokenType::Integer]
);
}
#[test]
fn test_keywords() {
assert_eq!(
tok_types("MATCH RETURN WHERE CREATE"),
vec![
TokenType::MATCH,
TokenType::RETURN,
TokenType::WHERE,
TokenType::CREATE,
]
);
assert_eq!(
tok_types("match return where create"),
vec![
TokenType::MATCH,
TokenType::RETURN,
TokenType::WHERE,
TokenType::CREATE,
]
);
}
#[test]
fn test_identifiers() {
let tokens = tokenize("n foo bar123 _x");
assert_eq!(tokens.len(), 4);
for t in &tokens {
assert_eq!(t.ty, TokenType::Identifier);
}
assert_eq!(tokens[0].literal, "n");
assert_eq!(tokens[1].literal, "foo");
assert_eq!(tokens[2].literal, "bar123");
assert_eq!(tokens[3].literal, "_x");
}
#[test]
fn test_integers() {
assert_eq!(tok_literals("42 0 100"), vec!["42", "0", "100"]);
assert_eq!(tok_types("42"), vec![TokenType::Integer]);
}
#[test]
fn test_hex_octal() {
assert_eq!(tok_types("0xFF"), vec![TokenType::Integer]);
assert_eq!(tok_literals("0xFF"), vec!["0xFF"]);
assert_eq!(tok_types("0o77"), vec![TokenType::Integer]);
assert_eq!(tok_literals("0o77"), vec!["0o77"]);
}
#[test]
fn test_floats() {
assert_eq!(tok_types("3.14"), vec![TokenType::Float]);
assert_eq!(tok_types("1e10"), vec![TokenType::Float]);
assert_eq!(tok_types("2.5E-3"), vec![TokenType::Float]);
assert_eq!(tok_types(".5"), vec![TokenType::Float]);
assert_eq!(tok_types(".4d"), vec![TokenType::Float]);
assert_eq!(tok_types("1e4D"), vec![TokenType::Float]);
}
#[test]
fn test_strings() {
let tokens = tokenize("'hello' \"world\"");
assert_eq!(tokens.len(), 2);
assert_eq!(tokens[0].ty, TokenType::String);
assert_eq!(tokens[0].literal, "hello");
assert_eq!(tokens[1].ty, TokenType::String);
assert_eq!(tokens[1].literal, "world");
}
#[test]
fn test_string_escapes() {
let tokens = tokenize(r"'hello\nworld'");
assert_eq!(tokens[0].literal, "hello\nworld");
let tokens = tokenize(r"'tab\there'");
assert_eq!(tokens[0].literal, "tab\there");
let tokens = tokenize(r"'escaped\\slash'");
assert_eq!(tokens[0].literal, "escaped\\slash");
}
#[test]
fn test_unicode_escape() {
let tokens = tokenize(r"'\u0041'");
assert_eq!(tokens[0].literal, "A");
}
#[test]
fn test_interpolation_escapes_match_string_escapes() {
let tokens = tokenize(r"s'a\nb\tc'");
assert_eq!(tokens[0].ty, TokenType::InterpString);
assert_eq!(tokens[0].literal, r"a\nb\tc");
let tokens = tokenize(r"s'\u0041{1}'");
assert_eq!(tokens[0].literal, r"\u0041{1}");
let tokens = tokenize(r"s'\{x\}'");
assert_eq!(tokens[0].literal, r"\{x\}");
let tokens = tokenize(r"s'\q'");
assert_eq!(tokens[0].literal, "\\q");
}
#[test]
fn test_backtick_identifier() {
let tokens = tokenize("`my label`");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0].ty, TokenType::Identifier);
assert_eq!(tokens[0].literal, "my label");
}
#[test]
fn test_backtick_identifier_doubled_backtick_is_escaped() {
let tokens = tokenize("`a``b`");
assert_eq!(tokens.len(), 1, "one identifier token: {tokens:?}");
assert_eq!(tokens[0].ty, TokenType::Identifier);
assert_eq!(tokens[0].literal, "a`b");
assert_eq!(tokenize("`x```")[0].literal, "x`");
assert_eq!(tokenize("```y`")[0].literal, "`y");
let two = tokenize("`a`b");
assert_eq!(two.len(), 2);
assert_eq!(two[0].literal, "a");
assert_eq!(two[1].literal, "b");
}
#[test]
fn test_parameter() {
let tokens = tokenize("$name $value");
assert_eq!(tokens.len(), 2);
assert_eq!(tokens[0].ty, TokenType::Parameter);
assert_eq!(tokens[0].literal, "name");
assert_eq!(tokens[1].literal, "value");
}
#[test]
fn test_line_comment() {
let _tokens = tokenize("MATCH // this is a comment\nRETURN");
assert_eq!(
tok_types("MATCH // comment\nRETURN"),
vec![TokenType::MATCH, TokenType::RETURN]
);
}
#[test]
fn test_block_comment() {
assert_eq!(
tok_types("MATCH /* block */ RETURN"),
vec![TokenType::MATCH, TokenType::RETURN]
);
}
#[test]
fn test_position_tracking() {
let tokens = tokenize("MATCH (n)\nRETURN n");
assert_eq!(tokens[0].line, 1);
assert_eq!(tokens[0].col, 1);
let ret_tok = tokens.iter().find(|t| t.ty == TokenType::RETURN).unwrap();
assert_eq!(ret_tok.line, 2);
assert_eq!(ret_tok.col, 1);
}
#[test]
fn test_full_query() {
let tokens = tokenize("MATCH (n:Person {name: 'Alice'})-[r:KNOWS]->(m) RETURN m.name");
assert!(tokens.len() > 10);
assert_eq!(tokens[0].ty, TokenType::MATCH);
}
#[test]
fn test_unterminated_string() {
let tokens = tokenize("'hello");
assert_eq!(tokens[0].ty, TokenType::Error);
assert_eq!(tokens[0].literal, "unterminated string");
}
#[test]
fn test_interpolation_embed_escapes_pass_through_raw() {
let tokens = tokenize(r"s'{coalesce('a\'b','c')}'");
assert_eq!(tokens[0].ty, TokenType::InterpString);
assert_eq!(tokens[0].literal, r"{coalesce('a\'b','c')}");
let tokens = tokenize(r"s'{coalesce('x\{y','z')}'");
assert_eq!(tokens[0].literal, r"{coalesce('x\{y','z')}");
}
#[test]
fn test_interpolation_escapes_pass_through_raw() {
let tokens = tokenize(r"s'a\nb\tc'");
assert_eq!(tokens[0].ty, TokenType::InterpString);
assert_eq!(tokens[0].literal, r"a\nb\tc");
let tokens = tokenize(r"s'\u0041{1}'");
assert_eq!(tokens[0].literal, r"\u0041{1}");
let tokens = tokenize(r"s'\{x\}'");
assert_eq!(tokens[0].literal, r"\{x\}");
let tokens = tokenize(r"s'\q'");
assert_eq!(tokens[0].literal, r"\q");
}
#[test]
fn test_interpolation_escaped_backslash_before_embed() {
assert_eq!(tokenize(r"s'\\{1}'")[0].literal, r"\\{1}");
assert_eq!(tokenize(r"s'\{1}'")[0].literal, r"\{1}");
assert_eq!(tokenize(r"s'a\\b{x}'")[0].literal, r"a\\b{x}");
}
#[test]
fn test_interpolation_backtick_name_in_embed() {
assert_eq!(tokenize(r"s'{`a}b`}'")[0].ty, TokenType::InterpString);
assert_eq!(tokenize(r"s'{`a}b`}'")[0].literal, r"{`a}b`}");
assert_eq!(tokenize(r#"s'{`x'y`}'"#)[0].literal, r"{`x'y`}");
}
#[test]
fn test_backtick_identifier_non_ascii_roundtrip() {
let tokens = tokenize("`Größe`");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0].ty, TokenType::Identifier);
assert_eq!(tokens[0].literal, "Größe");
assert_eq!(tokens[0].literal.as_bytes(), "Größe".as_bytes());
}
#[test]
fn test_unicode_escape_invalid_codepoint_is_error() {
for input in [r"'\uD800'", r"'\uDFFF'"] {
let tokens = tokenize(input);
assert_eq!(tokens[0].ty, TokenType::Error, "{input}");
assert!(
tokens[0].literal.contains("invalid unicode escape"),
"{input}: {}",
tokens[0].literal
);
}
}
#[test]
fn test_unicode_escape_window_cut_mid_character_is_error() {
let tokens = tokenize("'\u{5c}u00€'");
assert_eq!(tokens[0].ty, TokenType::Error);
assert!(
tokens[0].literal.contains("invalid unicode escape"),
"{}",
tokens[0].literal
);
assert_eq!(tokens[0].detail, Some("InvalidUnicodeLiteral"));
let tokens = tokenize(r"'\uGGGG'");
assert_eq!(tokens[0].ty, TokenType::Error);
assert_eq!(
tokens[0].literal,
r"invalid unicode escape sequence: \uGGGG"
);
assert_eq!(tokens[0].detail, Some("InvalidUnicodeLiteral"));
}
#[test]
fn test_unicode_escape_surrogate_pair_is_one_codepoint() {
let tokens = tokenize("'\\uD800\\uDC00'");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0].ty, TokenType::String);
assert_eq!(tokens[0].literal, "\u{10000}");
for input in [r"'\uD800\uD800'", r"'\uD800x'", r"'\uDC00\uD800'"] {
let tokens = tokenize(input);
assert_eq!(tokens[0].ty, TokenType::Error, "{input}");
}
}
#[test]
fn test_escape_fallback_preserves_non_ascii() {
let tokens = tokenize("'\\é'");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0].ty, TokenType::String);
assert_eq!(tokens[0].literal, "\\é");
let tokens = tokenize("'é\\n'");
assert_eq!(tokens[0].literal, "é\n");
}
#[test]
fn test_unterminated_block_comment_is_error() {
let tokens = tokenize("RETURN 1 /* DETACH DELETE n");
assert_eq!(tokens.len(), 3);
assert_eq!(tokens[0].ty, TokenType::RETURN);
assert_eq!(tokens[1].ty, TokenType::Integer);
assert_eq!(tokens[2].ty, TokenType::Error);
assert_eq!(tokens[2].literal, "unterminated block comment");
assert_eq!(
tok_types("RETURN 1 /* done */"),
vec![TokenType::RETURN, TokenType::Integer]
);
}
#[test]
fn test_unicode_identifier_lexes_as_one_token() {
for (query, ident) in [
("RETURN 1 AS Éx", "Éx"),
("RETURN 1 AS xÉ", "xÉ"),
("RETURN 1 AS xé", "xé"),
("RETURN 1 AS x€", "x€"), ("RETURN 1 AS x名前", "x名前"), ("RETURN 1 AS x\u{0301}", "x\u{0301}"), ("RETURN 1 AS x\u{08BE}", "x\u{08BE}"), ("RETURN 1 AS \u{203F}x", "\u{203F}x"), ("RETURN 1 AS \u{017F}KIP", "\u{017F}KIP"), ] {
let tokens = tokenize(query);
let last = tokens.last().expect("at least one token");
assert_eq!(last.ty, TokenType::Identifier, "{query:?}: {tokens:?}");
assert_eq!(last.literal, ident, "{query:?}");
}
let tokens = tokenize("RETURN $名前 AS v");
let param = tokens
.iter()
.find(|t| t.ty == TokenType::Parameter)
.unwrap_or_else(|| panic!("expected a parameter token: {tokens:?}"));
assert_eq!(param.literal, "名前");
}
#[test]
fn test_unicode_identifier_start_rejects_non_start_characters() {
for (query, ch) in [
("RETURN 1 AS 😀x", '😀'),
("RETURN 1 AS §x", '§'),
("RETURN 1 AS €x", '€'),
("RETURN 1 AS ·x", '·'),
("RETURN 1 AS x·", '·'), ] {
let tokens = tokenize(query);
let err = tokens
.iter()
.find(|t| t.ty == TokenType::Error)
.unwrap_or_else(|| panic!("{query:?}: expected an error token"));
assert_eq!(err.detail, Some("InvalidUnicodeCharacter"), "{query:?}");
assert!(err.literal.contains(ch), "{query:?}: {:?}", err.literal);
assert!(
tokens
.iter()
.any(|t| t.ty == TokenType::Identifier && t.literal.contains('x')),
"{query:?}: {tokens:?}"
);
}
}
#[test]
fn test_parameter_start_rejects_part_only_characters() {
for (query, bad) in [("RETURN $€ AS v", "€"), ("RETURN $€a AS v", "€a")] {
let tokens = tokenize(query);
let err = tokens
.iter()
.find(|t| t.ty == TokenType::Error)
.unwrap_or_else(|| panic!("{query:?} must produce an error token"));
assert_eq!(err.detail, Some("InvalidParameterName"), "{query:?}");
assert!(
err.literal.contains(bad),
"{query:?}: literal must name {bad:?}: {:?}",
err.literal
);
assert!(
err.literal
.contains("expected an identifier or an integer value"),
"{query:?}: message shape: {:?}",
err.literal
);
}
}
#[test]
fn test_parameter_start_accepts_digits_and_identifier_starts() {
for query in [
"RETURN $1 AS v",
"RETURN $1x AS v",
"RETURN $名前 AS v",
"RETURN $café AS v",
] {
let tokens = tokenize(query);
assert!(
!tokens.iter().any(|t| t.ty == TokenType::Error),
"{query:?} must lex clean: {tokens:?}"
);
}
}
#[test]
fn test_parameter_dynamic_opener_lexes_clean() {
let tokens = tokenize("MATCH (n:$(x)) RETURN n");
assert!(
!tokens.iter().any(|t| t.ty == TokenType::Error),
"the dynamic opener must lex clean: {tokens:?}"
);
}
#[test]
fn test_bare_parameter_rejected() {
let tokens = tokenize("RETURN $");
let err = tokens
.iter()
.find(|t| t.ty == TokenType::Error)
.unwrap_or_else(|| panic!("bare $ must produce an error token"));
assert_eq!(err.detail, Some("InvalidParameterName"));
assert!(
err.literal
.contains("expected an identifier or an integer value"),
"message shape: {:?}",
err.literal
);
}
#[test]
fn test_ascii_fast_path_long_identifier_runs() {
for name in [
"n",
"abcdefgh",
"abcdefghi",
"abcdefghijklmnopqrst",
"a1_b2_c3_d4_e5_f6_g7",
] {
let query = format!("RETURN 1 AS {name}");
let tokens = tokenize(&query);
let last = tokens
.last()
.unwrap_or_else(|| panic!("{query:?}: at least one token"));
assert_eq!(last.ty, TokenType::Identifier, "{query:?}: {tokens:?}");
assert_eq!(last.literal, name, "{query:?}");
}
}
#[test]
fn test_ascii_fast_path_stops_at_non_ascii_and_resumes() {
for name in [
"aaaaaaaaéaaaaaaaa",
"éaaaaaaaaaaaaaaaa",
"aaaaaaaaaaaaaaaaé",
"aé_bÉ1€x",
] {
let query = format!("RETURN 1 AS {name}");
let tokens = tokenize(&query);
let last = tokens
.last()
.unwrap_or_else(|| panic!("{query:?}: at least one token"));
assert_eq!(last.ty, TokenType::Identifier, "{query:?}: {tokens:?}");
assert_eq!(last.literal, name, "{query:?}");
}
let tokens = tokenize("RETURN $aaaaaaaaébbbb AS v");
let param = tokens
.iter()
.find(|t| t.ty == TokenType::Parameter)
.unwrap_or_else(|| panic!("expected a parameter token: {tokens:?}"));
assert_eq!(param.literal, "aaaaaaaaébbbb");
}
#[test]
fn test_ascii_fast_path_positions_after_batched_runs() {
let tokens = tokenize("MATCH (n)\n \t \r\n RETURN n.name");
let ret = tokens
.iter()
.find(|t| t.ty == TokenType::RETURN)
.unwrap_or_else(|| panic!("expected RETURN: {tokens:?}"));
assert_eq!((ret.line, ret.col), (3, 4), "{tokens:?}");
let tokens = tokenize("MATCH (n)\n RETURN");
let ret = tokens
.iter()
.find(|t| t.ty == TokenType::RETURN)
.unwrap_or_else(|| panic!("expected RETURN: {tokens:?}"));
assert_eq!((ret.line, ret.col), (2, 11), "{tokens:?}");
let tokens = tokenize("RETURN abcdefghijklmnop, 1");
let int = tokens
.iter()
.find(|t| t.ty == TokenType::Integer)
.unwrap_or_else(|| panic!("expected the integer: {tokens:?}"));
assert_eq!((int.line, int.col), (1, 26), "{tokens:?}");
let tokens = tokenize("RETURN aé, 1");
let int = tokens
.iter()
.find(|t| t.ty == TokenType::Integer)
.unwrap_or_else(|| panic!("expected the integer: {tokens:?}"));
assert_eq!((int.line, int.col), (1, 13), "{tokens:?}");
}
#[test]
fn test_ascii_fast_path_heavy_whitespace_matches_single_spaces() {
let thin = tokenize("MATCH (n:Person)-[:KNOWS]->(m) WHERE n.age > 21 RETURN m.name");
let wide = tokenize(
"MATCH (n:Person)\n\t-\t[:KNOWS]->\r\n (m) WHERE\tn.age\t> 21 RETURN m.name",
);
let shape = |ts: &[Token<'_>]| {
ts.iter()
.map(|t| (t.ty, t.literal.to_string()))
.collect::<Vec<_>>()
};
assert_eq!(shape(&thin), shape(&wide));
assert!(
!wide.iter().any(|t| t.ty == TokenType::Error),
"the wide form must lex clean: {wide:?}"
);
}