1#[cfg(wrap_proc_macro)]
2use crate::imp;
3#[cfg(span_locations)]
4use crate::location::LineColumn;
5use crate::parse::{self, Cursor};
6use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut};
7use crate::{Delimiter, Spacing, TokenTree};
8#[cfg(all(span_locations, not(fuzzing)))]
9use alloc::collections::BTreeMap;
10#[cfg(all(span_locations, not(fuzzing)))]
11use core::cell::RefCell;
12#[cfg(span_locations)]
13use core::cmp;
14use core::fmt::{self, Debug, Display, Write};
15#[cfg(span_locations)]
16use core::ops::Range;
17use core::ops::RangeBounds;
18use core::str;
19#[cfg(feature = "proc-macro")]
20use core::str::FromStr;
21use std::ffi::CStr;
22#[cfg(wrap_proc_macro)]
23use std::panic;
24#[cfg(procmacro2_semver_exempt)]
25use std::path::PathBuf;
26
27pub fn force() {
30 #[cfg(wrap_proc_macro)]
31 crate::detection::force_fallback();
32}
33
34pub fn unforce() {
37 #[cfg(wrap_proc_macro)]
38 crate::detection::unforce_fallback();
39}
40
41#[derive(Clone)]
42pub(crate) struct TokenStream {
43 inner: RcVec<TokenTree>,
44}
45
46#[derive(Debug)]
47pub(crate) struct LexError {
48 pub(crate) span: Span,
49}
50
51impl LexError {
52 pub(crate) fn span(&self) -> Span {
53 self.span
54 }
55
56 pub(crate) fn call_site() -> Self {
57 LexError {
58 span: Span::call_site(),
59 }
60 }
61}
62
63impl TokenStream {
64 pub(crate) fn new() -> Self {
65 TokenStream {
66 inner: RcVecBuilder::new().build(),
67 }
68 }
69
70 pub(crate) fn from_str_checked(src: &str) -> Result<Self, LexError> {
71 let mut cursor = get_cursor(src);
73
74 const BYTE_ORDER_MARK: &str = "\u{feff}";
76 if cursor.starts_with(BYTE_ORDER_MARK) {
77 cursor = cursor.advance(BYTE_ORDER_MARK.len());
78 }
79
80 parse::token_stream(cursor)
81 }
82
83 #[cfg(feature = "proc-macro")]
84 pub(crate) fn from_str_unchecked(src: &str) -> Self {
85 Self::from_str_checked(src).unwrap()
86 }
87
88 pub(crate) fn is_empty(&self) -> bool {
89 self.inner.len() == 0
90 }
91
92 fn take_inner(mut self) -> RcVecBuilder<TokenTree> {
93 let mut value = RcVecBuilder::new().build();
94 core::mem::swap(&mut self.inner, &mut value);
95 value.make_owned()
96 }
97}
98
99fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) {
100 match token {
102 TokenTree::Literal(crate::Literal {
103 #[cfg(wrap_proc_macro)]
104 inner: crate::imp::Literal::Fallback(literal),
105 #[cfg(not(wrap_proc_macro))]
106 inner: literal,
107 ..
108 }) if literal.repr.starts_with('-') => {
109 push_negative_literal(vec, literal);
110 }
111 _ => vec.push(token),
112 }
113
114 #[cold]
115 fn push_negative_literal(mut vec: RcVecMut<TokenTree>, mut literal: Literal) {
116 literal.repr.remove(0);
117 let mut punct = crate::Punct::new('-', Spacing::Alone);
118 punct.set_span(crate::Span::_new_fallback(literal.span));
119 vec.push(TokenTree::Punct(punct));
120 vec.push(TokenTree::Literal(crate::Literal::_new_fallback(literal)));
121 }
122}
123
124impl Drop for TokenStream {
126 fn drop(&mut self) {
127 let mut stack = Vec::new();
128 let mut current = match self.inner.get_mut() {
129 Some(inner) => inner.take().into_iter(),
130 None => return,
131 };
132 loop {
133 while let Some(token) = current.next() {
134 let group = match token {
135 TokenTree::Group(group) => group.inner,
136 _ => continue,
137 };
138 #[cfg(wrap_proc_macro)]
139 let group = match group {
140 crate::imp::Group::Fallback(group) => group,
141 crate::imp::Group::Compiler(_) => continue,
142 };
143 let mut group = group;
144 if let Some(inner) = group.stream.inner.get_mut() {
145 stack.push(current);
146 current = inner.take().into_iter();
147 }
148 }
149 match stack.pop() {
150 Some(next) => current = next,
151 None => return,
152 }
153 }
154 }
155}
156
157pub(crate) struct TokenStreamBuilder {
158 inner: RcVecBuilder<TokenTree>,
159}
160
161impl TokenStreamBuilder {
162 pub(crate) fn new() -> Self {
163 TokenStreamBuilder {
164 inner: RcVecBuilder::new(),
165 }
166 }
167
168 pub(crate) fn with_capacity(cap: usize) -> Self {
169 TokenStreamBuilder {
170 inner: RcVecBuilder::with_capacity(cap),
171 }
172 }
173
174 pub(crate) fn push_token_from_parser(&mut self, tt: TokenTree) {
175 self.inner.push(tt);
176 }
177
178 pub(crate) fn build(self) -> TokenStream {
179 TokenStream {
180 inner: self.inner.build(),
181 }
182 }
183}
184
185#[cfg(span_locations)]
186fn get_cursor(src: &str) -> Cursor {
187 #[cfg(fuzzing)]
188 return Cursor { rest: src, off: 1 };
189
190 #[cfg(not(fuzzing))]
192 SOURCE_MAP.with(|sm| {
193 let mut sm = sm.borrow_mut();
194 let span = sm.add_file(src);
195 Cursor {
196 rest: src,
197 off: span.lo,
198 }
199 })
200}
201
202#[cfg(not(span_locations))]
203fn get_cursor(src: &str) -> Cursor {
204 Cursor { rest: src }
205}
206
207impl Display for LexError {
208 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
209 f.write_str("cannot parse string into token stream")
210 }
211}
212
213impl Display for TokenStream {
214 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
215 let mut joint = false;
216 for (i, tt) in self.inner.iter().enumerate() {
217 if i != 0 && !joint {
218 write!(f, " ")?;
219 }
220 joint = false;
221 match tt {
222 TokenTree::Group(tt) => Display::fmt(tt, f),
223 TokenTree::Ident(tt) => Display::fmt(tt, f),
224 TokenTree::Punct(tt) => {
225 joint = tt.spacing() == Spacing::Joint;
226 Display::fmt(tt, f)
227 }
228 TokenTree::Literal(tt) => Display::fmt(tt, f),
229 }?;
230 }
231
232 Ok(())
233 }
234}
235
236impl Debug for TokenStream {
237 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
238 f.write_str("TokenStream ")?;
239 f.debug_list().entries(self.clone()).finish()
240 }
241}
242
243#[cfg(feature = "proc-macro")]
244impl From<proc_macro::TokenStream> for TokenStream {
245 fn from(inner: proc_macro::TokenStream) -> Self {
246 TokenStream::from_str_unchecked(&inner.to_string())
247 }
248}
249
250#[cfg(feature = "proc-macro")]
251impl From<TokenStream> for proc_macro::TokenStream {
252 fn from(inner: TokenStream) -> Self {
253 proc_macro::TokenStream::from_str_unchecked(&inner.to_string())
254 }
255}
256
257impl From<TokenTree> for TokenStream {
258 fn from(tree: TokenTree) -> Self {
259 let mut stream = RcVecBuilder::new();
260 push_token_from_proc_macro(stream.as_mut(), tree);
261 TokenStream {
262 inner: stream.build(),
263 }
264 }
265}
266
267impl FromIterator<TokenTree> for TokenStream {
268 fn from_iter<I: IntoIterator<Item = TokenTree>>(tokens: I) -> Self {
269 let mut stream = TokenStream::new();
270 stream.extend(tokens);
271 stream
272 }
273}
274
275impl FromIterator<TokenStream> for TokenStream {
276 fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
277 let mut v = RcVecBuilder::new();
278
279 for stream in streams {
280 v.extend(stream.take_inner());
281 }
282
283 TokenStream { inner: v.build() }
284 }
285}
286
287impl Extend<TokenTree> for TokenStream {
288 fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, tokens: I) {
289 let mut vec = self.inner.make_mut();
290 tokens
291 .into_iter()
292 .for_each(|token| push_token_from_proc_macro(vec.as_mut(), token));
293 }
294}
295
296impl Extend<TokenStream> for TokenStream {
297 fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
298 self.inner.make_mut().extend(streams.into_iter().flatten());
299 }
300}
301
302pub(crate) type TokenTreeIter = RcVecIntoIter<TokenTree>;
303
304impl IntoIterator for TokenStream {
305 type Item = TokenTree;
306 type IntoIter = TokenTreeIter;
307
308 fn into_iter(self) -> TokenTreeIter {
309 self.take_inner().into_iter()
310 }
311}
312
313#[cfg(all(span_locations, not(fuzzing)))]
314thread_local! {
315 static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
316 files: vec![FileInfo {
319 source_text: String::new(),
320 span: Span { lo: 0, hi: 0 },
321 lines: vec![0],
322 char_index_to_byte_offset: BTreeMap::new(),
323 }],
324 });
325}
326
327#[cfg(span_locations)]
328pub(crate) fn invalidate_current_thread_spans() {
329 #[cfg(not(fuzzing))]
330 SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1));
331}
332
333#[cfg(all(span_locations, not(fuzzing)))]
334struct FileInfo {
335 source_text: String,
336 span: Span,
337 lines: Vec<usize>,
338 char_index_to_byte_offset: BTreeMap<usize, usize>,
339}
340
341#[cfg(all(span_locations, not(fuzzing)))]
342impl FileInfo {
343 fn offset_line_column(&self, offset: usize) -> LineColumn {
344 assert!(self.span_within(Span {
345 lo: offset as u32,
346 hi: offset as u32,
347 }));
348 let offset = offset - self.span.lo as usize;
349 match self.lines.binary_search(&offset) {
350 Ok(found) => LineColumn {
351 line: found + 1,
352 column: 0,
353 },
354 Err(idx) => LineColumn {
355 line: idx,
356 column: offset - self.lines[idx - 1],
357 },
358 }
359 }
360
361 fn span_within(&self, span: Span) -> bool {
362 span.lo >= self.span.lo && span.hi <= self.span.hi
363 }
364
365 fn byte_range(&mut self, span: Span) -> Range<usize> {
366 let lo_char = (span.lo - self.span.lo) as usize;
367
368 let (&last_char_index, &last_byte_offset) = self
372 .char_index_to_byte_offset
373 .range(..=lo_char)
374 .next_back()
375 .unwrap_or((&0, &0));
376
377 let lo_byte = if last_char_index == lo_char {
378 last_byte_offset
379 } else {
380 let total_byte_offset = match self.source_text[last_byte_offset..]
381 .char_indices()
382 .nth(lo_char - last_char_index)
383 {
384 Some((additional_offset, _ch)) => last_byte_offset + additional_offset,
385 None => self.source_text.len(),
386 };
387 self.char_index_to_byte_offset
388 .insert(lo_char, total_byte_offset);
389 total_byte_offset
390 };
391
392 let trunc_lo = &self.source_text[lo_byte..];
393 let char_len = (span.hi - span.lo) as usize;
394 lo_byte..match trunc_lo.char_indices().nth(char_len) {
395 Some((offset, _ch)) => lo_byte + offset,
396 None => self.source_text.len(),
397 }
398 }
399
400 fn source_text(&mut self, span: Span) -> String {
401 let byte_range = self.byte_range(span);
402 self.source_text[byte_range].to_owned()
403 }
404}
405
406#[cfg(all(span_locations, not(fuzzing)))]
409fn lines_offsets(s: &str) -> (usize, Vec<usize>) {
410 let mut lines = vec![0];
411 let mut total = 0;
412
413 for ch in s.chars() {
414 total += 1;
415 if ch == '\n' {
416 lines.push(total);
417 }
418 }
419
420 (total, lines)
421}
422
423#[cfg(all(span_locations, not(fuzzing)))]
424struct SourceMap {
425 files: Vec<FileInfo>,
426}
427
428#[cfg(all(span_locations, not(fuzzing)))]
429impl SourceMap {
430 fn next_start_pos(&self) -> u32 {
431 self.files.last().unwrap().span.hi + 1
436 }
437
438 fn add_file(&mut self, src: &str) -> Span {
439 let (len, lines) = lines_offsets(src);
440 let lo = self.next_start_pos();
441 let span = Span {
442 lo,
443 hi: lo + (len as u32),
444 };
445
446 self.files.push(FileInfo {
447 source_text: src.to_owned(),
448 span,
449 lines,
450 char_index_to_byte_offset: BTreeMap::new(),
452 });
453
454 span
455 }
456
457 #[cfg(procmacro2_semver_exempt)]
458 fn filepath(&self, span: Span) -> String {
459 for (i, file) in self.files.iter().enumerate() {
460 if file.span_within(span) {
461 return if i == 0 {
462 "<unspecified>".to_owned()
463 } else {
464 format!("<parsed string {}>", i)
465 };
466 }
467 }
468 unreachable!("Invalid span with no related FileInfo!");
469 }
470
471 fn fileinfo(&self, span: Span) -> &FileInfo {
472 for file in &self.files {
473 if file.span_within(span) {
474 return file;
475 }
476 }
477 unreachable!("Invalid span with no related FileInfo!");
478 }
479
480 fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo {
481 for file in &mut self.files {
482 if file.span_within(span) {
483 return file;
484 }
485 }
486 unreachable!("Invalid span with no related FileInfo!");
487 }
488}
489
490#[derive(Clone, Copy, PartialEq, Eq)]
491pub(crate) struct Span {
492 #[cfg(span_locations)]
493 pub(crate) lo: u32,
494 #[cfg(span_locations)]
495 pub(crate) hi: u32,
496}
497
498impl Span {
499 #[cfg(not(span_locations))]
500 pub(crate) fn call_site() -> Self {
501 Span {}
502 }
503
504 #[cfg(span_locations)]
505 pub(crate) fn call_site() -> Self {
506 Span { lo: 0, hi: 0 }
507 }
508
509 pub(crate) fn mixed_site() -> Self {
510 Span::call_site()
511 }
512
513 #[cfg(procmacro2_semver_exempt)]
514 pub(crate) fn def_site() -> Self {
515 Span::call_site()
516 }
517
518 pub(crate) fn resolved_at(&self, _other: Span) -> Span {
519 *self
523 }
524
525 pub(crate) fn located_at(&self, other: Span) -> Span {
526 other
527 }
528
529 #[cfg(span_locations)]
530 pub(crate) fn byte_range(&self) -> Range<usize> {
531 #[cfg(fuzzing)]
532 return 0..0;
533
534 #[cfg(not(fuzzing))]
535 {
536 if self.is_call_site() {
537 0..0
538 } else {
539 SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self))
540 }
541 }
542 }
543
544 #[cfg(span_locations)]
545 pub(crate) fn start(&self) -> LineColumn {
546 #[cfg(fuzzing)]
547 return LineColumn { line: 0, column: 0 };
548
549 #[cfg(not(fuzzing))]
550 SOURCE_MAP.with(|sm| {
551 let sm = sm.borrow();
552 let fi = sm.fileinfo(*self);
553 fi.offset_line_column(self.lo as usize)
554 })
555 }
556
557 #[cfg(span_locations)]
558 pub(crate) fn end(&self) -> LineColumn {
559 #[cfg(fuzzing)]
560 return LineColumn { line: 0, column: 0 };
561
562 #[cfg(not(fuzzing))]
563 SOURCE_MAP.with(|sm| {
564 let sm = sm.borrow();
565 let fi = sm.fileinfo(*self);
566 fi.offset_line_column(self.hi as usize)
567 })
568 }
569
570 #[cfg(procmacro2_semver_exempt)]
571 pub(crate) fn file(&self) -> String {
572 #[cfg(fuzzing)]
573 return "<unspecified>".to_owned();
574
575 #[cfg(not(fuzzing))]
576 SOURCE_MAP.with(|sm| {
577 let sm = sm.borrow();
578 sm.filepath(*self)
579 })
580 }
581
582 #[cfg(procmacro2_semver_exempt)]
583 pub(crate) fn local_file(&self) -> Option<PathBuf> {
584 None
585 }
586
587 #[cfg(not(span_locations))]
588 pub(crate) fn join(&self, _other: Span) -> Option<Span> {
589 Some(Span {})
590 }
591
592 #[cfg(span_locations)]
593 pub(crate) fn join(&self, other: Span) -> Option<Span> {
594 #[cfg(fuzzing)]
595 return {
596 let _ = other;
597 None
598 };
599
600 #[cfg(not(fuzzing))]
601 SOURCE_MAP.with(|sm| {
602 let sm = sm.borrow();
603 if !sm.fileinfo(*self).span_within(other) {
605 return None;
606 }
607 Some(Span {
608 lo: cmp::min(self.lo, other.lo),
609 hi: cmp::max(self.hi, other.hi),
610 })
611 })
612 }
613
614 #[cfg(not(span_locations))]
615 pub(crate) fn source_text(&self) -> Option<String> {
616 None
617 }
618
619 #[cfg(span_locations)]
620 pub(crate) fn source_text(&self) -> Option<String> {
621 #[cfg(fuzzing)]
622 return None;
623
624 #[cfg(not(fuzzing))]
625 {
626 if self.is_call_site() {
627 None
628 } else {
629 Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self)))
630 }
631 }
632 }
633
634 #[cfg(not(span_locations))]
635 pub(crate) fn first_byte(self) -> Self {
636 self
637 }
638
639 #[cfg(span_locations)]
640 pub(crate) fn first_byte(self) -> Self {
641 Span {
642 lo: self.lo,
643 hi: cmp::min(self.lo.saturating_add(1), self.hi),
644 }
645 }
646
647 #[cfg(not(span_locations))]
648 pub(crate) fn last_byte(self) -> Self {
649 self
650 }
651
652 #[cfg(span_locations)]
653 pub(crate) fn last_byte(self) -> Self {
654 Span {
655 lo: cmp::max(self.hi.saturating_sub(1), self.lo),
656 hi: self.hi,
657 }
658 }
659
660 #[cfg(span_locations)]
661 fn is_call_site(&self) -> bool {
662 self.lo == 0 && self.hi == 0
663 }
664}
665
666impl Debug for Span {
667 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
668 #[cfg(span_locations)]
669 return write!(f, "bytes({}..{})", self.lo, self.hi);
670
671 #[cfg(not(span_locations))]
672 write!(f, "Span")
673 }
674}
675
676pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
677 #[cfg(span_locations)]
678 {
679 if span.is_call_site() {
680 return;
681 }
682 }
683
684 if cfg!(span_locations) {
685 debug.field("span", &span);
686 }
687}
688
689#[derive(Clone)]
690pub(crate) struct Group {
691 delimiter: Delimiter,
692 stream: TokenStream,
693 span: Span,
694}
695
696impl Group {
697 pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
698 Group {
699 delimiter,
700 stream,
701 span: Span::call_site(),
702 }
703 }
704
705 pub(crate) fn delimiter(&self) -> Delimiter {
706 self.delimiter
707 }
708
709 pub(crate) fn stream(&self) -> TokenStream {
710 self.stream.clone()
711 }
712
713 pub(crate) fn span(&self) -> Span {
714 self.span
715 }
716
717 pub(crate) fn span_open(&self) -> Span {
718 self.span.first_byte()
719 }
720
721 pub(crate) fn span_close(&self) -> Span {
722 self.span.last_byte()
723 }
724
725 pub(crate) fn set_span(&mut self, span: Span) {
726 self.span = span;
727 }
728}
729
730impl Display for Group {
731 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
739 let (open, close) = match self.delimiter {
740 Delimiter::Parenthesis => ("(", ")"),
741 Delimiter::Brace => ("{ ", "}"),
742 Delimiter::Bracket => ("[", "]"),
743 Delimiter::None => ("", ""),
744 };
745
746 f.write_str(open)?;
747 Display::fmt(&self.stream, f)?;
748 if self.delimiter == Delimiter::Brace && !self.stream.inner.is_empty() {
749 f.write_str(" ")?;
750 }
751 f.write_str(close)?;
752
753 Ok(())
754 }
755}
756
757impl Debug for Group {
758 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
759 let mut debug = fmt.debug_struct("Group");
760 debug.field("delimiter", &self.delimiter);
761 debug.field("stream", &self.stream);
762 debug_span_field_if_nontrivial(&mut debug, self.span);
763 debug.finish()
764 }
765}
766
767#[derive(Clone)]
768pub(crate) struct Ident {
769 sym: Box<str>,
770 span: Span,
771 raw: bool,
772}
773
774impl Ident {
775 #[track_caller]
776 pub(crate) fn new_checked(string: &str, span: Span) -> Self {
777 validate_ident(string);
778 Ident::new_unchecked(string, span)
779 }
780
781 pub(crate) fn new_unchecked(string: &str, span: Span) -> Self {
782 Ident {
783 sym: Box::from(string),
784 span,
785 raw: false,
786 }
787 }
788
789 #[track_caller]
790 pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self {
791 validate_ident_raw(string);
792 Ident::new_raw_unchecked(string, span)
793 }
794
795 pub(crate) fn new_raw_unchecked(string: &str, span: Span) -> Self {
796 Ident {
797 sym: Box::from(string),
798 span,
799 raw: true,
800 }
801 }
802
803 pub(crate) fn span(&self) -> Span {
804 self.span
805 }
806
807 pub(crate) fn set_span(&mut self, span: Span) {
808 self.span = span;
809 }
810}
811
812pub(crate) fn is_ident_start(c: char) -> bool {
813 c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
814}
815
816pub(crate) fn is_ident_continue(c: char) -> bool {
817 unicode_xid::UnicodeXID::is_xid_continue(c)
818}
819
820#[track_caller]
821fn validate_ident(string: &str) {
822 if string.is_empty() {
823 panic!("Ident is not allowed to be empty; use Option<Ident>");
824 }
825
826 if string.bytes().all(|digit| b'0' <= digit && digit <= b'9') {
827 panic!("Ident cannot be a number; use Literal instead");
828 }
829
830 fn ident_ok(string: &str) -> bool {
831 let mut chars = string.chars();
832 let first = chars.next().unwrap();
833 if !is_ident_start(first) {
834 return false;
835 }
836 for ch in chars {
837 if !is_ident_continue(ch) {
838 return false;
839 }
840 }
841 true
842 }
843
844 if !ident_ok(string) {
845 panic!("{:?} is not a valid Ident", string);
846 }
847}
848
849#[track_caller]
850fn validate_ident_raw(string: &str) {
851 validate_ident(string);
852
853 match string {
854 "_" | "super" | "self" | "Self" | "crate" => {
855 panic!("`r#{}` cannot be a raw identifier", string);
856 }
857 _ => {}
858 }
859}
860
861impl PartialEq for Ident {
862 fn eq(&self, other: &Ident) -> bool {
863 self.sym == other.sym && self.raw == other.raw
864 }
865}
866
867impl<T> PartialEq<T> for Ident
868where
869 T: ?Sized + AsRef<str>,
870{
871 fn eq(&self, other: &T) -> bool {
872 let other = other.as_ref();
873 if self.raw {
874 other.starts_with("r#") && *self.sym == other[2..]
875 } else {
876 *self.sym == *other
877 }
878 }
879}
880
881impl Display for Ident {
882 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
883 if self.raw {
884 f.write_str("r#")?;
885 }
886 Display::fmt(&self.sym, f)
887 }
888}
889
890#[allow(clippy::missing_fields_in_debug)]
891impl Debug for Ident {
892 #[cfg(not(span_locations))]
894 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
895 let mut debug = f.debug_tuple("Ident");
896 debug.field(&format_args!("{}", self));
897 debug.finish()
898 }
899
900 #[cfg(span_locations)]
905 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
906 let mut debug = f.debug_struct("Ident");
907 debug.field("sym", &format_args!("{}", self));
908 debug_span_field_if_nontrivial(&mut debug, self.span);
909 debug.finish()
910 }
911}
912
913#[derive(Clone)]
914pub(crate) struct Literal {
915 pub(crate) repr: String,
916 span: Span,
917}
918
919macro_rules! suffixed_numbers {
920 ($($name:ident => $kind:ident,)*) => ($(
921 pub(crate) fn $name(n: $kind) -> Literal {
922 Literal::_new(format!(concat!("{}", stringify!($kind)), n))
923 }
924 )*)
925}
926
927macro_rules! unsuffixed_numbers {
928 ($($name:ident => $kind:ident,)*) => ($(
929 pub(crate) fn $name(n: $kind) -> Literal {
930 Literal::_new(n.to_string())
931 }
932 )*)
933}
934
935impl Literal {
936 pub(crate) fn _new(repr: String) -> Self {
937 Literal {
938 repr,
939 span: Span::call_site(),
940 }
941 }
942
943 pub(crate) fn from_str_checked(repr: &str) -> Result<Self, LexError> {
944 let mut cursor = get_cursor(repr);
945 #[cfg(span_locations)]
946 let lo = cursor.off;
947
948 let negative = cursor.starts_with_char('-');
949 if negative {
950 cursor = cursor.advance(1);
951 if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) {
952 return Err(LexError::call_site());
953 }
954 }
955
956 if let Ok((rest, mut literal)) = parse::literal(cursor) {
957 if rest.is_empty() {
958 if negative {
959 literal.repr.insert(0, '-');
960 }
961 literal.span = Span {
962 #[cfg(span_locations)]
963 lo,
964 #[cfg(span_locations)]
965 hi: rest.off,
966 };
967 return Ok(literal);
968 }
969 }
970 Err(LexError::call_site())
971 }
972
973 pub(crate) fn from_str_unchecked(repr: &str) -> Self {
974 Self::from_str_checked(repr).unwrap()
975 }
976
977 suffixed_numbers! {
978 u8_suffixed => u8,
979 u16_suffixed => u16,
980 u32_suffixed => u32,
981 u64_suffixed => u64,
982 u128_suffixed => u128,
983 usize_suffixed => usize,
984 i8_suffixed => i8,
985 i16_suffixed => i16,
986 i32_suffixed => i32,
987 i64_suffixed => i64,
988 i128_suffixed => i128,
989 isize_suffixed => isize,
990
991 f32_suffixed => f32,
992 f64_suffixed => f64,
993 }
994
995 unsuffixed_numbers! {
996 u8_unsuffixed => u8,
997 u16_unsuffixed => u16,
998 u32_unsuffixed => u32,
999 u64_unsuffixed => u64,
1000 u128_unsuffixed => u128,
1001 usize_unsuffixed => usize,
1002 i8_unsuffixed => i8,
1003 i16_unsuffixed => i16,
1004 i32_unsuffixed => i32,
1005 i64_unsuffixed => i64,
1006 i128_unsuffixed => i128,
1007 isize_unsuffixed => isize,
1008 }
1009
1010 pub(crate) fn f32_unsuffixed(f: f32) -> Literal {
1011 let mut s = f.to_string();
1012 if !s.contains('.') {
1013 s.push_str(".0");
1014 }
1015 Literal::_new(s)
1016 }
1017
1018 pub(crate) fn f64_unsuffixed(f: f64) -> Literal {
1019 let mut s = f.to_string();
1020 if !s.contains('.') {
1021 s.push_str(".0");
1022 }
1023 Literal::_new(s)
1024 }
1025
1026 pub(crate) fn string(string: &str) -> Literal {
1027 let mut repr = String::with_capacity(string.len() + 2);
1028 repr.push('"');
1029 escape_utf8(string, &mut repr);
1030 repr.push('"');
1031 Literal::_new(repr)
1032 }
1033
1034 pub(crate) fn character(ch: char) -> Literal {
1035 let mut repr = String::new();
1036 repr.push('\'');
1037 if ch == '"' {
1038 repr.push(ch);
1040 } else {
1041 repr.extend(ch.escape_debug());
1042 }
1043 repr.push('\'');
1044 Literal::_new(repr)
1045 }
1046
1047 pub(crate) fn byte_character(byte: u8) -> Literal {
1048 let mut repr = "b'".to_string();
1049 #[allow(clippy::match_overlapping_arm)]
1050 match byte {
1051 b'\0' => repr.push_str(r"\0"),
1052 b'\t' => repr.push_str(r"\t"),
1053 b'\n' => repr.push_str(r"\n"),
1054 b'\r' => repr.push_str(r"\r"),
1055 b'\'' => repr.push_str(r"\'"),
1056 b'\\' => repr.push_str(r"\\"),
1057 b'\x20'..=b'\x7E' => repr.push(byte as char),
1058 _ => {
1059 let _ = write!(repr, r"\x{:02X}", byte);
1060 }
1061 }
1062 repr.push('\'');
1063 Literal::_new(repr)
1064 }
1065
1066 pub(crate) fn byte_string(bytes: &[u8]) -> Literal {
1067 let mut repr = "b\"".to_string();
1068 let mut bytes = bytes.iter();
1069 while let Some(&b) = bytes.next() {
1070 #[allow(clippy::match_overlapping_arm)]
1071 match b {
1072 b'\0' => repr.push_str(match bytes.as_slice().first() {
1073 Some(b'0'..=b'7') => r"\x00",
1075 _ => r"\0",
1076 }),
1077 b'\t' => repr.push_str(r"\t"),
1078 b'\n' => repr.push_str(r"\n"),
1079 b'\r' => repr.push_str(r"\r"),
1080 b'"' => repr.push_str("\\\""),
1081 b'\\' => repr.push_str(r"\\"),
1082 b'\x20'..=b'\x7E' => repr.push(b as char),
1083 _ => {
1084 let _ = write!(repr, r"\x{:02X}", b);
1085 }
1086 }
1087 }
1088 repr.push('"');
1089 Literal::_new(repr)
1090 }
1091
1092 pub(crate) fn c_string(string: &CStr) -> Literal {
1093 let mut repr = "c\"".to_string();
1094 let mut bytes = string.to_bytes();
1095 while !bytes.is_empty() {
1096 let (valid, invalid) = match str::from_utf8(bytes) {
1097 Ok(all_valid) => {
1098 bytes = b"";
1099 (all_valid, bytes)
1100 }
1101 Err(utf8_error) => {
1102 let (valid, rest) = bytes.split_at(utf8_error.valid_up_to());
1103 let valid = str::from_utf8(valid).unwrap();
1104 let invalid = utf8_error
1105 .error_len()
1106 .map_or(rest, |error_len| &rest[..error_len]);
1107 bytes = &bytes[valid.len() + invalid.len()..];
1108 (valid, invalid)
1109 }
1110 };
1111 escape_utf8(valid, &mut repr);
1112 for &byte in invalid {
1113 let _ = write!(repr, r"\x{:02X}", byte);
1114 }
1115 }
1116 repr.push('"');
1117 Literal::_new(repr)
1118 }
1119
1120 pub(crate) fn span(&self) -> Span {
1121 self.span
1122 }
1123
1124 pub(crate) fn set_span(&mut self, span: Span) {
1125 self.span = span;
1126 }
1127
1128 pub(crate) fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
1129 #[cfg(not(span_locations))]
1130 {
1131 let _ = range;
1132 None
1133 }
1134
1135 #[cfg(span_locations)]
1136 {
1137 use core::ops::Bound;
1138
1139 let lo = match range.start_bound() {
1140 Bound::Included(start) => {
1141 let start = u32::try_from(*start).ok()?;
1142 self.span.lo.checked_add(start)?
1143 }
1144 Bound::Excluded(start) => {
1145 let start = u32::try_from(*start).ok()?;
1146 self.span.lo.checked_add(start)?.checked_add(1)?
1147 }
1148 Bound::Unbounded => self.span.lo,
1149 };
1150 let hi = match range.end_bound() {
1151 Bound::Included(end) => {
1152 let end = u32::try_from(*end).ok()?;
1153 self.span.lo.checked_add(end)?.checked_add(1)?
1154 }
1155 Bound::Excluded(end) => {
1156 let end = u32::try_from(*end).ok()?;
1157 self.span.lo.checked_add(end)?
1158 }
1159 Bound::Unbounded => self.span.hi,
1160 };
1161 if lo <= hi && hi <= self.span.hi {
1162 Some(Span { lo, hi })
1163 } else {
1164 None
1165 }
1166 }
1167 }
1168}
1169
1170impl Display for Literal {
1171 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1172 Display::fmt(&self.repr, f)
1173 }
1174}
1175
1176impl Debug for Literal {
1177 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1178 let mut debug = fmt.debug_struct("Literal");
1179 debug.field("lit", &format_args!("{}", self.repr));
1180 debug_span_field_if_nontrivial(&mut debug, self.span);
1181 debug.finish()
1182 }
1183}
1184
1185fn escape_utf8(string: &str, repr: &mut String) {
1186 let mut chars = string.chars();
1187 while let Some(ch) = chars.next() {
1188 if ch == '\0' {
1189 repr.push_str(
1190 if chars
1191 .as_str()
1192 .starts_with(|next| '0' <= next && next <= '7')
1193 {
1194 r"\x00"
1196 } else {
1197 r"\0"
1198 },
1199 );
1200 } else if ch == '\'' {
1201 repr.push(ch);
1203 } else {
1204 repr.extend(ch.escape_debug());
1205 }
1206 }
1207}
1208
1209#[cfg(feature = "proc-macro")]
1210pub(crate) trait FromStr2: FromStr<Err = proc_macro::LexError> {
1211 #[cfg(wrap_proc_macro)]
1212 fn valid(src: &str) -> bool;
1213
1214 #[cfg(wrap_proc_macro)]
1215 fn from_str_checked(src: &str) -> Result<Self, imp::LexError> {
1216 if !Self::valid(src) {
1220 return Err(imp::LexError::CompilerPanic);
1221 }
1222
1223 match panic::catch_unwind(|| Self::from_str(src)) {
1225 Ok(Ok(ok)) => Ok(ok),
1226 Ok(Err(lex)) => Err(imp::LexError::Compiler(lex)),
1227 Err(_panic) => Err(imp::LexError::CompilerPanic),
1228 }
1229 }
1230
1231 fn from_str_unchecked(src: &str) -> Self {
1232 Self::from_str(src).unwrap()
1233 }
1234}
1235
1236#[cfg(feature = "proc-macro")]
1237impl FromStr2 for proc_macro::TokenStream {
1238 #[cfg(wrap_proc_macro)]
1239 fn valid(src: &str) -> bool {
1240 TokenStream::from_str_checked(src).is_ok()
1241 }
1242}
1243
1244#[cfg(feature = "proc-macro")]
1245impl FromStr2 for proc_macro::Literal {
1246 #[cfg(wrap_proc_macro)]
1247 fn valid(src: &str) -> bool {
1248 Literal::from_str_checked(src).is_ok()
1249 }
1250}