1use std::marker::PhantomData;
2use std::ptr::NonNull;
3
4use crate::ast_names::{AstName, AstNameTable, LexemeType};
5use crate::location::{Location, Position};
6
7mod string;
8mod token;
9
10pub(crate) use string::{fixup_string_bytes, multiline_string_bytes};
11pub use token::{LexedToken, QuoteStyle, ReservedWord, Token, TokenKind};
12
13#[derive(Clone)]
14pub struct Lexer<'source, 'ast, 'name, 'names>
15where
16 'name: 'ast,
17{
18 source: &'source [u8],
19 offset: usize,
20 line: u32,
21 line_offset: usize,
22 current: LexedToken<'source, 'ast>,
23 previous_location: Location,
24 brace_stack: Vec<BraceType>,
25 names: NonNull<AstNameTable<'name>>,
26 read_names: bool,
27 _names: PhantomData<&'names mut AstNameTable<'name>>,
28}
29
30#[derive(Clone, Copy)]
31struct LexerSnapshot<'source, 'ast> {
32 offset: usize,
33 line: u32,
34 line_offset: usize,
35 current: LexedToken<'source, 'ast>,
36 previous_location: Location,
37 brace_stack_size: usize,
38 brace_type: BraceType,
39}
40
41impl<'source, 'ast, 'name, 'names> Lexer<'source, 'ast, 'name, 'names>
42where
43 'name: 'ast,
44{
45 pub fn new(source: &'source [u8], names: &'names mut AstNameTable<'name>) -> Self {
46 Self::with_position(source, names, Position::zero())
47 }
48
49 pub fn with_position(
50 source: &'source [u8],
51 names: &'names mut AstNameTable<'name>,
52 start_position: Position,
53 ) -> Self {
54 let current = LexedToken {
55 token: Token::Eof,
56 start: 0,
57 end: 0,
58 location: Location::new(start_position, start_position),
59 };
60 Self {
61 source,
62 offset: 0,
63 line: start_position.line,
64 line_offset: 0usize.wrapping_sub(start_position.column as usize),
65 current,
66 previous_location: Location::new(start_position, start_position),
67 brace_stack: Vec::new(),
68 names: NonNull::from(names),
69 read_names: true,
70 _names: PhantomData,
71 }
72 }
73
74 pub fn source(&self) -> &'source [u8] {
75 self.source
76 }
77
78 pub(crate) fn interpolation_brace_is_open(&self) -> bool {
79 self.brace_stack.last() == Some(&BraceType::InterpolatedString)
80 }
81
82 pub fn set_read_names(&mut self, read_names: bool) {
83 self.read_names = read_names;
84 }
85
86 pub fn next_token_span(&mut self, skip_comments: bool) -> (Token<'source, 'ast>, usize, usize) {
87 self.advance_lexeme(skip_comments);
88 (self.current.token, self.current.start, self.current.end)
89 }
90
91 pub fn next_lexeme(&mut self, skip_comments: bool) -> LexedToken<'source, 'ast> {
92 self.advance_lexeme(skip_comments);
93 self.current
94 }
95
96 pub fn advance_lexeme(&mut self, skip_comments: bool) {
97 self.advance_lexeme_with_previous(skip_comments, true);
98 }
99
100 pub(crate) fn advance_lexeme_with_previous(
101 &mut self,
102 skip_comments: bool,
103 mut update_previous_location: bool,
104 ) {
105 loop {
106 while is_space(self.peekch()) {
107 self.consume_any();
108 }
109
110 if update_previous_location {
111 self.previous_location = self.current.location;
112 }
113
114 self.current = self.read_next();
115 update_previous_location = false;
116
117 if skip_comments
118 && matches!(
119 self.current.token,
120 Token::Comment(_) | Token::BlockComment(_)
121 )
122 {
123 continue;
124 }
125
126 return;
127 }
128 }
129
130 pub fn current_lexeme(&self) -> LexedToken<'source, 'ast> {
131 self.current
132 }
133
134 pub fn current_token(&self) -> Token<'source, 'ast> {
135 self.current.token
136 }
137
138 pub fn current_location(&self) -> Location {
139 self.current.location
140 }
141
142 pub fn current_span(&self) -> (usize, usize) {
143 (self.current.start, self.current.end)
144 }
145
146 pub fn previous_location(&self) -> Location {
147 self.previous_location
148 }
149
150 pub fn lookahead_token(&mut self, skip_comments: bool) -> Token<'source, 'ast> {
151 self.lookahead_lexeme(skip_comments).token
152 }
153
154 pub fn lookahead_nth_token(&mut self, n: usize, skip_comments: bool) -> Token<'source, 'ast> {
155 self.lookahead_nth_lexeme(n, skip_comments).token
156 }
157
158 pub fn lookahead_lexeme(&mut self, skip_comments: bool) -> LexedToken<'source, 'ast> {
159 let snapshot = self.snapshot();
160 let result = self.next_lexeme(skip_comments);
161 self.restore(snapshot);
162 result
163 }
164
165 pub fn lookahead_nth_lexeme(
166 &mut self,
167 n: usize,
168 skip_comments: bool,
169 ) -> LexedToken<'source, 'ast> {
170 debug_assert!(n > 0);
171
172 let snapshot = self.snapshot();
173 let mut result = self.next_lexeme(skip_comments);
174 for _ in 1..n {
175 result = self.next_lexeme(skip_comments);
176 }
177 self.restore(snapshot);
178 result
179 }
180
181 fn snapshot(&self) -> LexerSnapshot<'source, 'ast> {
182 LexerSnapshot {
183 offset: self.offset,
184 line: self.line,
185 line_offset: self.line_offset,
186 current: self.current,
187 previous_location: self.previous_location,
188 brace_stack_size: self.brace_stack.len(),
189 brace_type: self
190 .brace_stack
191 .last()
192 .copied()
193 .unwrap_or(BraceType::Normal),
194 }
195 }
196
197 fn restore(&mut self, snapshot: LexerSnapshot<'source, 'ast>) {
198 self.offset = snapshot.offset;
199 self.line = snapshot.line;
200 self.line_offset = snapshot.line_offset;
201 self.current = snapshot.current;
202 self.previous_location = snapshot.previous_location;
203
204 if self.brace_stack.len() < snapshot.brace_stack_size {
205 self.brace_stack.push(snapshot.brace_type);
206 } else if self.brace_stack.len() > snapshot.brace_stack_size {
207 self.brace_stack.pop();
208 }
209 }
210
211 #[inline(always)]
212 fn peekch(&self) -> u8 {
213 if self.offset < self.source.len() {
214 unsafe { *self.source.get_unchecked(self.offset) }
215 } else {
216 0
217 }
218 }
219
220 #[inline(always)]
221 fn peekch_n(&self, lookahead: usize) -> u8 {
222 let offset = self.offset + lookahead;
223 if offset < self.source.len() {
224 unsafe { *self.source.get_unchecked(offset) }
225 } else {
226 0
227 }
228 }
229
230 #[inline(always)]
231 fn position(&self) -> Position {
232 Position::new(self.line, self.offset.wrapping_sub(self.line_offset) as u32)
233 }
234
235 #[inline(always)]
236 fn consume(&mut self) {
237 debug_assert!(self.peekch() != b'\n');
238 self.offset += 1;
239 }
240
241 #[inline(always)]
242 fn consume_any(&mut self) {
243 if is_newline(self.peekch()) {
244 self.line += 1;
245 self.line_offset = self.offset + 1;
246 }
247
248 self.offset += 1;
249 }
250
251 fn names_mut(&mut self) -> &mut AstNameTable<'name> {
252 unsafe { self.names.as_mut() }
253 }
254
255 fn name_with_type_bytes(&mut self, name: &[u8]) -> (AstName<'ast>, LexemeType) {
256 let (name, kind) = if self.read_names {
257 self.names_mut().get_or_add_bytes_with_type(name)
258 } else {
259 self.names_mut()
260 .get_bytes_with_type(name)
261 .unwrap_or((AstName::empty_key(), LexemeType::Name))
262 };
263 (name.narrow(), kind)
264 }
265
266 pub(crate) fn intern_name_bytes(&mut self, name: &[u8]) -> AstName<'ast> {
267 let name = if self.read_names {
268 self.names_mut().get_or_add_bytes(name)
269 } else {
270 self.names_mut()
271 .get_bytes(name)
272 .unwrap_or_else(AstName::empty_key)
273 };
274 name.narrow()
275 }
276
277 #[inline(always)]
278 fn lexed_from_range(
279 &self,
280 start: Position,
281 start_offset: usize,
282 end_offset: usize,
283 token: Token<'source, 'ast>,
284 ) -> LexedToken<'source, 'ast> {
285 LexedToken {
286 token,
287 start: start_offset,
288 end: end_offset,
289 location: Location::new(start, self.position()),
290 }
291 }
292
293 fn read_comment_body(&mut self) -> LexedToken<'source, 'ast> {
294 let start = self.position();
295 let start_offset = self.offset;
296
297 debug_assert!(self.peekch() == b'-' && self.peekch_n(1) == b'-');
298 self.consume();
299 self.consume();
300
301 let body_offset = self.offset;
302
303 if self.peekch() == b'[' {
304 let sep = self.skip_long_separator();
305 if sep >= 0 {
306 return self.read_long_string(
307 start,
308 start_offset,
309 sep as usize,
310 LongTokenKind::BlockComment,
311 );
312 }
313 }
314
315 while self.peekch() != 0 && self.peekch() != b'\r' && !is_newline(self.peekch()) {
316 self.consume();
317 }
318
319 self.lexed_from_range(
320 start,
321 start_offset,
322 self.offset,
323 Token::Comment(&self.source[body_offset..self.offset]),
324 )
325 }
326
327 fn skip_long_separator(&mut self) -> i32 {
328 let start = self.peekch();
329 debug_assert!(start == b'[' || start == b']');
330 self.consume();
331
332 let mut count = 0i32;
333 while self.peekch() == b'=' {
334 self.consume();
335 count += 1;
336 }
337
338 if start == self.peekch() {
339 count
340 } else {
341 -count - 1
342 }
343 }
344
345 fn read_long_string(
346 &mut self,
347 start: Position,
348 start_offset: usize,
349 sep: usize,
350 kind: LongTokenKind,
351 ) -> LexedToken<'source, 'ast> {
352 debug_assert!(self.peekch() == b'[');
353 self.consume();
354
355 let body_offset = self.offset;
356
357 while self.peekch() != 0 {
358 if self.peekch() == b']' {
359 if self.skip_long_separator() == sep as i32 {
360 debug_assert!(self.peekch() == b']');
361 self.consume();
362
363 let end_offset = self.offset - sep - 2;
364 let token = match kind {
365 LongTokenKind::RawString => Token::RawString {
366 value: &self.source[body_offset..end_offset],
367 block_depth: sep,
368 },
369 LongTokenKind::BlockComment => Token::BlockComment(sep),
370 };
371
372 return self.lexed_from_range(start, start_offset, self.offset, token);
373 }
374 } else {
375 self.consume_any();
376 }
377 }
378
379 let token = match kind {
380 LongTokenKind::RawString => Token::BrokenString,
381 LongTokenKind::BlockComment => Token::BrokenComment,
382 };
383 self.lexed_from_range(start, start_offset, self.offset, token)
384 }
385
386 fn read_backslash_in_string(&mut self) {
387 debug_assert!(self.peekch() == b'\\');
388 self.consume();
389
390 match self.peekch() {
391 b'\r' => {
392 self.consume();
393 if self.peekch() == b'\n' {
394 self.consume_any();
395 }
396 }
397 0 => {}
398 b'z' => {
399 self.consume();
400 while is_space(self.peekch()) {
401 self.consume_any();
402 }
403 }
404 _ => self.consume_any(),
405 }
406 }
407
408 fn read_quoted_string(&mut self) -> LexedToken<'source, 'ast> {
409 let start = self.position();
410 let start_offset = self.offset;
411 let delimiter = self.peekch();
412 debug_assert!(delimiter == b'\'' || delimiter == b'"');
413 self.consume();
414
415 let body_offset = self.offset;
416
417 while self.peekch() != delimiter {
418 match self.peekch() {
419 0 | b'\r' | b'\n' => {
420 return self.lexed_from_range(
421 start,
422 start_offset,
423 self.offset,
424 Token::BrokenString,
425 );
426 }
427 b'\\' => self.read_backslash_in_string(),
428 _ => self.consume(),
429 }
430 }
431
432 self.consume();
433
434 self.lexed_from_range(
435 start,
436 start_offset,
437 self.offset,
438 Token::QuotedString {
439 value: &self.source[body_offset..self.offset - 1],
440 quote_style: QuoteStyle::from_delimiter(delimiter),
441 },
442 )
443 }
444
445 fn read_interpolated_string_begin(&mut self) -> LexedToken<'source, 'ast> {
446 debug_assert!(self.peekch() == b'`');
447
448 let start = self.position();
449 let start_offset = self.offset;
450 self.consume();
451
452 self.read_interpolated_string_section(start, start_offset, InterpolatedSectionKind::Begin)
453 }
454
455 fn read_interpolated_string_section(
456 &mut self,
457 start: Position,
458 start_offset: usize,
459 kind: InterpolatedSectionKind,
460 ) -> LexedToken<'source, 'ast> {
461 let body_offset = self.offset;
462
463 while self.peekch() != b'`' {
464 match self.peekch() {
465 0 | b'\r' | b'\n' => {
466 return self.lexed_from_range(
467 start,
468 start_offset,
469 self.offset,
470 Token::BrokenString,
471 );
472 }
473 b'\\' => {
474 if self.peekch_n(1) == b'u' && self.peekch_n(2) == b'{' {
475 self.consume();
476 self.consume();
477 self.consume();
478 } else {
479 self.read_backslash_in_string();
480 }
481 }
482 b'{' => {
483 self.brace_stack.push(BraceType::InterpolatedString);
484
485 if self.peekch_n(1) == b'{' {
486 let token =
487 Token::BrokenInterpDoubleBrace(&self.source[body_offset..self.offset]);
488 self.consume();
489 self.consume();
490 return self.lexed_from_range(start, start_offset, self.offset, token);
491 }
492
493 self.consume();
494
495 let token = match kind {
496 InterpolatedSectionKind::Begin => {
497 Token::InterpStringBegin(&self.source[body_offset..self.offset - 1])
498 }
499 InterpolatedSectionKind::Mid => {
500 Token::InterpStringMid(&self.source[body_offset..self.offset - 1])
501 }
502 };
503
504 return self.lexed_from_range(start, start_offset, self.offset, token);
505 }
506 _ => self.consume(),
507 }
508 }
509
510 self.consume();
511
512 let token = match kind {
513 InterpolatedSectionKind::Begin => {
514 Token::InterpStringSimple(&self.source[body_offset..self.offset - 1])
515 }
516 InterpolatedSectionKind::Mid => {
517 Token::InterpStringEnd(&self.source[body_offset..self.offset - 1])
518 }
519 };
520
521 self.lexed_from_range(start, start_offset, self.offset, token)
522 }
523
524 #[inline(always)]
525 fn read_number(&mut self, start: Position, start_offset: usize) -> LexedToken<'source, 'ast> {
526 debug_assert!(is_digit(self.peekch()));
527
528 loop {
529 self.consume();
530 let ch = self.peekch();
531 if !(is_digit(ch) || ch == b'.' || ch == b'_') {
532 break;
533 }
534 }
535
536 if matches!(self.peekch(), b'e' | b'E') {
537 self.consume();
538
539 if matches!(self.peekch(), b'+' | b'-') {
540 self.consume();
541 }
542 }
543
544 while is_alpha(self.peekch()) || is_digit(self.peekch()) || self.peekch() == b'_' {
545 self.consume();
546 }
547
548 self.lexed_from_range(
549 start,
550 start_offset,
551 self.offset,
552 Token::Number(&self.source[start_offset..self.offset]),
553 )
554 }
555
556 #[inline(always)]
557 fn read_name(&mut self) -> (AstName<'ast>, LexemeType) {
558 debug_assert!(is_alpha(self.peekch()) || self.peekch() == b'_');
559
560 let start_offset = self.offset;
561 loop {
562 self.consume();
563 let ch = self.peekch();
564 if !(is_alpha(ch) || is_digit(ch) || ch == b'_') {
565 break;
566 }
567 }
568
569 self.name_with_type_bytes(&self.source[start_offset..self.offset])
570 }
571
572 fn read_attribute_name(&mut self) -> AstName<'ast> {
573 debug_assert!(is_alpha(self.peekch()) || self.peekch() == b'_');
574
575 let start_offset = self.offset;
576 loop {
577 self.consume();
578 let ch = self.peekch();
579 if !(is_alpha(ch) || is_digit(ch) || ch == b'_') {
580 break;
581 }
582 }
583
584 self.intern_name_bytes(&self.source[start_offset..self.offset])
585 }
586
587 fn read_utf8_error(&mut self) -> LexedToken<'source, 'ast> {
588 let start = self.position();
589 let start_offset = self.offset;
590 let first = self.peekch();
591
592 let (size, mut codepoint) = if (first & 0b1000_0000) == 0b0000_0000 {
593 (1usize, u32::from(first & 0x7f))
594 } else if (first & 0b1110_0000) == 0b1100_0000 {
595 (2usize, u32::from(first & 0b1_1111))
596 } else if (first & 0b1111_0000) == 0b1110_0000 {
597 (3usize, u32::from(first & 0b1111))
598 } else if (first & 0b1111_1000) == 0b1111_0000 {
599 (4usize, u32::from(first & 0b111))
600 } else {
601 self.consume();
602 return self.lexed_from_range(
603 start,
604 start_offset,
605 self.offset,
606 Token::BrokenUnicode { codepoint: 0 },
607 );
608 };
609
610 self.consume();
611
612 for _ in 1..size {
613 if (self.peekch() & 0b1100_0000) != 0b1000_0000 {
614 return self.lexed_from_range(
615 start,
616 start_offset,
617 self.offset,
618 Token::BrokenUnicode { codepoint: 0 },
619 );
620 }
621
622 codepoint = (codepoint << 6) | u32::from(self.peekch() & 0b0011_1111);
623 self.consume();
624 }
625
626 self.lexed_from_range(
627 start,
628 start_offset,
629 self.offset,
630 Token::BrokenUnicode { codepoint },
631 )
632 }
633
634 fn read_next(&mut self) -> LexedToken<'source, 'ast> {
635 let start = self.position();
636 let start_offset = self.offset;
637
638 match self.peekch() {
639 0 => self.lexed_from_range(start, start_offset, start_offset, Token::Eof),
640 b'-' => {
641 if self.peekch_n(1) == b'>' {
642 self.consume();
643 self.consume();
644 self.lexed_from_range(start, start_offset, self.offset, Token::SkinnyArrow)
645 } else if self.peekch_n(1) == b'=' {
646 self.consume();
647 self.consume();
648 self.lexed_from_range(start, start_offset, self.offset, Token::MinusEqual)
649 } else if self.peekch_n(1) == b'-' {
650 self.read_comment_body()
651 } else {
652 self.consume();
653 self.lexed_from_range(start, start_offset, self.offset, Token::Minus)
654 }
655 }
656 b'[' => {
657 let sep = self.skip_long_separator();
658 if sep >= 0 {
659 self.read_long_string(
660 start,
661 start_offset,
662 sep as usize,
663 LongTokenKind::RawString,
664 )
665 } else if sep == -1 {
666 self.lexed_from_range(start, start_offset, self.offset, Token::LeftBracket)
667 } else {
668 self.lexed_from_range(start, start_offset, self.offset, Token::BrokenString)
669 }
670 }
671 b'{' => {
672 self.consume();
673 if !self.brace_stack.is_empty() {
674 self.brace_stack.push(BraceType::Normal);
675 }
676 self.lexed_from_range(start, start_offset, self.offset, Token::LeftBrace)
677 }
678 b'}' => {
679 self.consume();
680
681 let Some(brace) = self.brace_stack.pop() else {
682 return self.lexed_from_range(
683 start,
684 start_offset,
685 self.offset,
686 Token::RightBrace,
687 );
688 };
689
690 if brace != BraceType::InterpolatedString {
691 return self.lexed_from_range(
692 start,
693 start_offset,
694 self.offset,
695 Token::RightBrace,
696 );
697 }
698
699 self.read_interpolated_string_section(
700 start,
701 start_offset,
702 InterpolatedSectionKind::Mid,
703 )
704 }
705 b'=' => {
706 self.consume();
707 if self.peekch() == b'=' {
708 self.consume();
709 self.lexed_from_range(start, start_offset, self.offset, Token::EqualEqual)
710 } else {
711 self.lexed_from_range(start, start_offset, self.offset, Token::Equal)
712 }
713 }
714 b'<' => {
715 self.consume();
716 if self.peekch() == b'=' {
717 self.consume();
718 self.lexed_from_range(start, start_offset, self.offset, Token::LessEqual)
719 } else {
720 self.lexed_from_range(start, start_offset, self.offset, Token::Less)
721 }
722 }
723 b'>' => {
724 self.consume();
725 if self.peekch() == b'=' {
726 self.consume();
727 self.lexed_from_range(start, start_offset, self.offset, Token::GreaterEqual)
728 } else {
729 self.lexed_from_range(start, start_offset, self.offset, Token::Greater)
730 }
731 }
732 b'~' => {
733 self.consume();
734 if self.peekch() == b'=' {
735 self.consume();
736 self.lexed_from_range(start, start_offset, self.offset, Token::TildeEqual)
737 } else {
738 self.lexed_from_range(start, start_offset, self.offset, Token::Error)
739 }
740 }
741 b'"' | b'\'' => self.read_quoted_string(),
742 b'`' => self.read_interpolated_string_begin(),
743 b'.' => {
744 self.consume();
745
746 if self.peekch() == b'.' {
747 self.consume();
748
749 if self.peekch() == b'.' {
750 self.consume();
751 self.lexed_from_range(start, start_offset, self.offset, Token::Ellipsis)
752 } else if self.peekch() == b'=' {
753 self.consume();
754 self.lexed_from_range(start, start_offset, self.offset, Token::DotDotEqual)
755 } else {
756 self.lexed_from_range(start, start_offset, self.offset, Token::DotDot)
757 }
758 } else if is_digit(self.peekch()) {
759 self.read_number(start, start_offset)
760 } else {
761 self.lexed_from_range(start, start_offset, self.offset, Token::Dot)
762 }
763 }
764 b'+' => {
765 self.consume();
766 if self.peekch() == b'=' {
767 self.consume();
768 self.lexed_from_range(start, start_offset, self.offset, Token::PlusEqual)
769 } else {
770 self.lexed_from_range(start, start_offset, self.offset, Token::Plus)
771 }
772 }
773 b'/' => {
774 self.consume();
775 match self.peekch() {
776 b'=' => {
777 self.consume();
778 self.lexed_from_range(start, start_offset, self.offset, Token::SlashEqual)
779 }
780 b'/' => {
781 self.consume();
782 if self.peekch() == b'=' {
783 self.consume();
784 self.lexed_from_range(
785 start,
786 start_offset,
787 self.offset,
788 Token::SlashSlashEqual,
789 )
790 } else {
791 self.lexed_from_range(
792 start,
793 start_offset,
794 self.offset,
795 Token::SlashSlash,
796 )
797 }
798 }
799 _ => self.lexed_from_range(start, start_offset, self.offset, Token::Slash),
800 }
801 }
802 b'*' => {
803 self.consume();
804 if self.peekch() == b'=' {
805 self.consume();
806 self.lexed_from_range(start, start_offset, self.offset, Token::StarEqual)
807 } else {
808 self.lexed_from_range(start, start_offset, self.offset, Token::Star)
809 }
810 }
811 b'%' => {
812 self.consume();
813 if self.peekch() == b'=' {
814 self.consume();
815 self.lexed_from_range(start, start_offset, self.offset, Token::PercentEqual)
816 } else {
817 self.lexed_from_range(start, start_offset, self.offset, Token::Percent)
818 }
819 }
820 b'^' => {
821 self.consume();
822 if self.peekch() == b'=' {
823 self.consume();
824 self.lexed_from_range(start, start_offset, self.offset, Token::CaretEqual)
825 } else {
826 self.lexed_from_range(start, start_offset, self.offset, Token::Caret)
827 }
828 }
829 b':' => {
830 self.consume();
831 if self.peekch() == b':' {
832 self.consume();
833 self.lexed_from_range(start, start_offset, self.offset, Token::DoubleColon)
834 } else {
835 self.lexed_from_range(start, start_offset, self.offset, Token::Colon)
836 }
837 }
838 b'(' => {
839 self.consume();
840 self.lexed_from_range(start, start_offset, self.offset, Token::LeftParen)
841 }
842 b')' => {
843 self.consume();
844 self.lexed_from_range(start, start_offset, self.offset, Token::RightParen)
845 }
846 b']' => {
847 self.consume();
848 self.lexed_from_range(start, start_offset, self.offset, Token::RightBracket)
849 }
850 b';' => {
851 self.consume();
852 self.lexed_from_range(start, start_offset, self.offset, Token::Semicolon)
853 }
854 b',' => {
855 self.consume();
856 self.lexed_from_range(start, start_offset, self.offset, Token::Comma)
857 }
858 b'#' => {
859 self.consume();
860 self.lexed_from_range(start, start_offset, self.offset, Token::Hash)
861 }
862 b'!' => {
863 self.consume();
864 self.lexed_from_range(start, start_offset, self.offset, Token::Bang)
865 }
866 b'?' => {
867 self.consume();
868 self.lexed_from_range(start, start_offset, self.offset, Token::Question)
869 }
870 b'&' => {
871 self.consume();
872 self.lexed_from_range(start, start_offset, self.offset, Token::Ampersand)
873 }
874 b'|' => {
875 self.consume();
876 self.lexed_from_range(start, start_offset, self.offset, Token::Pipe)
877 }
878 b'@' => {
879 if self.peekch_n(1) == b'[' {
880 self.consume();
881 self.consume();
882 self.lexed_from_range(start, start_offset, self.offset, Token::AttributeOpen)
883 } else {
884 self.consume();
885 let token = if is_alpha(self.peekch()) || self.peekch() == b'_' {
886 Token::Attribute(self.read_attribute_name())
887 } else {
888 Token::Attribute(AstName::from_static(""))
889 };
890 self.lexed_from_range(start, start_offset, self.offset, token)
891 }
892 }
893 byte if is_digit(byte) => self.read_number(start, start_offset),
894 byte if is_alpha(byte) || byte == b'_' => {
895 let (name, kind) = self.read_name();
896 let token = ReservedWord::from_lexeme_type(kind)
897 .map(Token::Reserved)
898 .unwrap_or(Token::Ident(name));
899 self.lexed_from_range(start, start_offset, self.offset, token)
900 }
901 byte if (byte & 0x80) != 0 => self.read_utf8_error(),
902 _ => {
903 self.consume();
904 self.lexed_from_range(start, start_offset, self.offset, Token::Error)
905 }
906 }
907 }
908}
909
910#[derive(Clone, Copy, PartialEq, Eq)]
911enum BraceType {
912 InterpolatedString,
913 Normal,
914}
915
916#[derive(Clone, Copy)]
917enum LongTokenKind {
918 RawString,
919 BlockComment,
920}
921
922#[derive(Clone, Copy)]
923enum InterpolatedSectionKind {
924 Begin,
925 Mid,
926}
927
928fn is_alpha(ch: u8) -> bool {
929 (ch | b' ').wrapping_sub(b'a') < 26
930}
931
932fn is_digit(ch: u8) -> bool {
933 ch.wrapping_sub(b'0') < 10
934}
935
936fn is_hex_digit(ch: u8) -> bool {
937 ch.wrapping_sub(b'0') < 10 || (ch | b' ').wrapping_sub(b'a') < 6
938}
939
940fn is_newline(ch: u8) -> bool {
941 ch == b'\n'
942}
943
944fn is_space(ch: u8) -> bool {
945 matches!(ch, b' ' | b'\t' | b'\r' | b'\n' | 0x0b | 0x0c)
946}
947
948fn unescape(ch: u8) -> u8 {
949 match ch {
950 b'a' => 0x07,
951 b'b' => 0x08,
952 b'f' => 0x0c,
953 b'n' => b'\n',
954 b'r' => b'\r',
955 b't' => b'\t',
956 b'v' => 0x0b,
957 other => other,
958 }
959}
960
961fn hex_value(ch: u8) -> Option<u8> {
962 match ch {
963 b'0'..=b'9' => Some(ch - b'0'),
964 b'a'..=b'f' => Some(ch - b'a' + 10),
965 b'A'..=b'F' => Some(ch - b'A' + 10),
966 _ => None,
967 }
968}
969
970fn to_utf8(data: &mut [u8], code: u32) -> Option<usize> {
971 match code {
972 0x0000..=0x007f => {
973 data[0] = code as u8;
974 Some(1)
975 }
976 0x0080..=0x07ff => {
977 data[0] = (0xc0 | (code >> 6)) as u8;
978 data[1] = (0x80 | (code & 0x3f)) as u8;
979 Some(2)
980 }
981 0x0800..=0xffff => {
982 data[0] = (0xe0 | (code >> 12)) as u8;
983 data[1] = (0x80 | ((code >> 6) & 0x3f)) as u8;
984 data[2] = (0x80 | (code & 0x3f)) as u8;
985 Some(3)
986 }
987 0x10000..=0x10ffff => {
988 data[0] = (0xf0 | (code >> 18)) as u8;
989 data[1] = (0x80 | ((code >> 12) & 0x3f)) as u8;
990 data[2] = (0x80 | ((code >> 6) & 0x3f)) as u8;
991 data[3] = (0x80 | (code & 0x3f)) as u8;
992 Some(4)
993 }
994 _ => None,
995 }
996}