1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
use super::*;
fn tokens_of(input: &str) -> Vec<Token> {
let mut lexer = Lexer::new(input);
lexer
.tokenize()
.into_iter()
.map(|t| t.token)
.filter(|t| *t != Token::EOF)
.collect()
}
#[test]
fn single_char_between_single_quotes_is_a_character_literal() {
// Plan 270 §"The rule" item 3: exactly one character between single
// quotes is a character literal, not a one-character identifier —
// this is why single-character quoted identifiers do not exist.
assert_eq!(tokens_of("'A'"), vec![Token::IntegerLiteral(65)]);
}
#[test]
fn possessive_single_apostrophe_form_lexes_identifier_then_s() {
// Plan 270 §5: `'name's` reads as the quoted identifier plus the
// possessive marker, not `Unknown function: s`. `length` is a
// contextual word (a synonym of `size` in the possessive dispatch
// only), so it lexes as an ordinary identifier, matching the
// plan's own canonical example (`'total items's length`).
assert_eq!(
tokens_of("'my nums's length"),
vec![
Token::Identifier("my nums".to_string()),
Token::Apostrophe,
Token::Identifier("s".to_string()),
Token::Identifier("length".to_string()),
]
);
}
#[test]
fn possessive_doubled_apostrophe_form_lexes_identically() {
// Plan 270 §5: `'name''s` is the pre-existing doubled-apostrophe
// path and must produce the exact same token stream as the new
// single-apostrophe possessive form, so the parser needs no new path.
assert_eq!(
tokens_of("'my nums''s length"),
tokens_of("'my nums's length")
);
}
#[test]
fn underscore_prefix_is_preserved_in_identifier() {
// Regression 1: `_str_eq` must lex as a single identifier with the
// leading underscore intact, not as the bare name `str_eq`.
assert_eq!(
tokens_of("_str_eq"),
vec![Token::Identifier("_str_eq".to_string())]
);
// Mid-word underscores continue to work.
assert_eq!(
tokens_of("my_helper"),
vec![Token::Identifier("my_helper".to_string())]
);
}