1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
use super::scanner::Scanner;
use super::TokenKind;
pub fn scan_string(scanner: &mut Scanner, start_char: char) -> (TokenKind, bool) {
let mut is_fstring = false;
let mut is_bytes = false;
// Check prefixes before the first quote
if start_char == 'f' || start_char == 'F' {
is_fstring = true;
}
if start_char == 'b' || start_char == 'B' {
is_bytes = true;
}
// We consumed the prefix char already, now check for more prefixes and the quote
let ch = scanner.peek().unwrap_or('\0');
let mut quote_char = ch;
if quote_char == '"' || quote_char == '\'' {
scanner.advance(); // consume first quote
} else {
let prefixes = if ch == 'r' || ch == 'R' {
scanner.advance();
scanner.peek().unwrap_or('\0')
} else if ch == 'f' || ch == 'F' {
is_fstring = true;
scanner.advance();
scanner.peek().unwrap_or('\0')
} else {
'\0'
};
quote_char = prefixes;
if quote_char != '"' && quote_char != '\'' {
return (TokenKind::StringLit, is_fstring);
}
scanner.advance();
}
// Check for triple quotes
let is_triple = if scanner.peek() == Some(quote_char) {
scanner.advance();
if scanner.peek() == Some(quote_char) {
scanner.advance();
true
} else {
// Was just two quotes in a row
let kind = if is_bytes {
TokenKind::BytesLit
} else {
TokenKind::StringLit
};
return (kind, is_fstring);
}
} else {
false
};
// Read string content
loop {
match scanner.peek() {
None => break,
Some('\\') => {
scanner.advance();
if scanner.peek().is_some() {
scanner.advance();
}
}
Some(ch) if ch == quote_char => {
if is_triple {
scanner.advance();
if scanner.peek() == Some(quote_char) {
scanner.advance();
if scanner.peek() == Some(quote_char) {
scanner.advance();
break;
}
}
} else {
scanner.advance();
break;
}
}
Some('{') if is_fstring && !is_triple => {
// Keep the complete f-string in one token. Expression-aware
// splitting belongs in the parser and must not leave the
// lexer stranded halfway through a string.
scanner.advance();
}
_ => {
scanner.advance();
}
}
}
let kind = if is_bytes {
TokenKind::BytesLit
} else if is_fstring {
TokenKind::FStringLit
} else {
TokenKind::StringLit
};
(kind, false)
}
pub fn scan_fstring_interior(scanner: &mut Scanner) -> (TokenKind, bool) {
// Read until matching }
let mut depth = 1u32;
loop {
match scanner.peek() {
None => break,
Some('{') => {
depth += 1;
scanner.advance();
}
Some('}') => {
depth -= 1;
scanner.advance();
if depth == 0 {
// Check if more string follows
match scanner.peek() {
Some('"') => {
let _saved_pos = scanner.position();
scanner.advance();
// Check for fstring continuation
if scanner.peek() == Some('"') {
scanner.advance();
}
// Return to string
let kind = TokenKind::FStringMid;
return (kind, true);
}
Some('\'') => {
scanner.advance();
if scanner.peek() == Some('\'') {
scanner.advance();
}
let kind = TokenKind::FStringMid;
return (kind, true);
}
_ => {
// Expression is done
let kind = TokenKind::FStringTail;
return (kind, false);
}
}
}
}
Some('\'') | Some('"') => {
scanner.advance();
}
Some(_ch) => {
scanner.advance();
}
}
}
(TokenKind::Error, false)
}