safe_proc_macro2/
fallback.rs

1#[cfg(wrap_proc_macro)]
2use crate::imp;
3#[cfg(span_locations)]
4use crate::location::LineColumn;
5use crate::parse::{self, Cursor};
6use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut};
7use crate::{Delimiter, Spacing, TokenTree};
8#[cfg(all(span_locations, not(fuzzing)))]
9use alloc::collections::BTreeMap;
10#[cfg(all(span_locations, not(fuzzing)))]
11use core::cell::RefCell;
12#[cfg(span_locations)]
13use core::cmp;
14use core::fmt::{self, Debug, Display, Write};
15#[cfg(span_locations)]
16use core::ops::Range;
17use core::ops::RangeBounds;
18use core::str;
19#[cfg(feature = "proc-macro")]
20use core::str::FromStr;
21use std::ffi::CStr;
22#[cfg(wrap_proc_macro)]
23use std::panic;
24#[cfg(procmacro2_semver_exempt)]
25use std::path::PathBuf;
26
27/// Force use of proc-macro2's fallback implementation of the API for now, even
28/// if the compiler's implementation is available.
29pub fn force() {
30    #[cfg(wrap_proc_macro)]
31    crate::detection::force_fallback();
32}
33
34/// Resume using the compiler's implementation of the proc macro API if it is
35/// available.
36pub fn unforce() {
37    #[cfg(wrap_proc_macro)]
38    crate::detection::unforce_fallback();
39}
40
41#[derive(Clone)]
42pub(crate) struct TokenStream {
43    inner: RcVec<TokenTree>,
44}
45
46#[derive(Debug)]
47pub(crate) struct LexError {
48    pub(crate) span: Span,
49}
50
51impl LexError {
52    pub(crate) fn span(&self) -> Span {
53        self.span
54    }
55
56    pub(crate) fn call_site() -> Self {
57        LexError {
58            span: Span::call_site(),
59        }
60    }
61}
62
63impl TokenStream {
64    pub(crate) fn new() -> Self {
65        TokenStream {
66            inner: RcVecBuilder::new().build(),
67        }
68    }
69
70    pub(crate) fn from_str_checked(src: &str) -> Result<Self, LexError> {
71        // Create a dummy file & add it to the source map
72        let mut cursor = get_cursor(src);
73
74        // Strip a byte order mark if present
75        const BYTE_ORDER_MARK: &str = "\u{feff}";
76        if cursor.starts_with(BYTE_ORDER_MARK) {
77            cursor = cursor.advance(BYTE_ORDER_MARK.len());
78        }
79
80        parse::token_stream(cursor)
81    }
82
83    #[cfg(feature = "proc-macro")]
84    pub(crate) fn from_str_unchecked(src: &str) -> Self {
85        Self::from_str_checked(src).unwrap()
86    }
87
88    pub(crate) fn is_empty(&self) -> bool {
89        self.inner.len() == 0
90    }
91
92    fn take_inner(mut self) -> RcVecBuilder<TokenTree> {
93        let mut value = RcVecBuilder::new().build();
94        core::mem::swap(&mut self.inner, &mut value);
95        value.make_owned()
96    }
97}
98
99fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) {
100    // https://github.com/dtolnay/proc-macro2/issues/235
101    match token {
102        TokenTree::Literal(crate::Literal {
103            #[cfg(wrap_proc_macro)]
104                inner: crate::imp::Literal::Fallback(literal),
105            #[cfg(not(wrap_proc_macro))]
106                inner: literal,
107            ..
108        }) if literal.repr.starts_with('-') => {
109            push_negative_literal(vec, literal);
110        }
111        _ => vec.push(token),
112    }
113
114    #[cold]
115    fn push_negative_literal(mut vec: RcVecMut<TokenTree>, mut literal: Literal) {
116        literal.repr.remove(0);
117        let mut punct = crate::Punct::new('-', Spacing::Alone);
118        punct.set_span(crate::Span::_new_fallback(literal.span));
119        vec.push(TokenTree::Punct(punct));
120        vec.push(TokenTree::Literal(crate::Literal::_new_fallback(literal)));
121    }
122}
123
124// Nonrecursive to prevent stack overflow.
125impl Drop for TokenStream {
126    fn drop(&mut self) {
127        let mut stack = Vec::new();
128        let mut current = match self.inner.get_mut() {
129            Some(inner) => inner.take().into_iter(),
130            None => return,
131        };
132        loop {
133            while let Some(token) = current.next() {
134                let group = match token {
135                    TokenTree::Group(group) => group.inner,
136                    _ => continue,
137                };
138                #[cfg(wrap_proc_macro)]
139                let group = match group {
140                    crate::imp::Group::Fallback(group) => group,
141                    crate::imp::Group::Compiler(_) => continue,
142                };
143                let mut group = group;
144                if let Some(inner) = group.stream.inner.get_mut() {
145                    stack.push(current);
146                    current = inner.take().into_iter();
147                }
148            }
149            match stack.pop() {
150                Some(next) => current = next,
151                None => return,
152            }
153        }
154    }
155}
156
157pub(crate) struct TokenStreamBuilder {
158    inner: RcVecBuilder<TokenTree>,
159}
160
161impl TokenStreamBuilder {
162    pub(crate) fn new() -> Self {
163        TokenStreamBuilder {
164            inner: RcVecBuilder::new(),
165        }
166    }
167
168    pub(crate) fn with_capacity(cap: usize) -> Self {
169        TokenStreamBuilder {
170            inner: RcVecBuilder::with_capacity(cap),
171        }
172    }
173
174    pub(crate) fn push_token_from_parser(&mut self, tt: TokenTree) {
175        self.inner.push(tt);
176    }
177
178    pub(crate) fn build(self) -> TokenStream {
179        TokenStream {
180            inner: self.inner.build(),
181        }
182    }
183}
184
185#[cfg(span_locations)]
186fn get_cursor(src: &str) -> Cursor {
187    #[cfg(fuzzing)]
188    return Cursor { rest: src, off: 1 };
189
190    // Create a dummy file & add it to the source map
191    #[cfg(not(fuzzing))]
192    SOURCE_MAP.with(|sm| {
193        let mut sm = sm.borrow_mut();
194        let span = sm.add_file(src);
195        Cursor {
196            rest: src,
197            off: span.lo,
198        }
199    })
200}
201
202#[cfg(not(span_locations))]
203fn get_cursor(src: &str) -> Cursor {
204    Cursor { rest: src }
205}
206
207impl Display for LexError {
208    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
209        f.write_str("cannot parse string into token stream")
210    }
211}
212
213impl Display for TokenStream {
214    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
215        let mut joint = false;
216        for (i, tt) in self.inner.iter().enumerate() {
217            if i != 0 && !joint {
218                write!(f, " ")?;
219            }
220            joint = false;
221            match tt {
222                TokenTree::Group(tt) => Display::fmt(tt, f),
223                TokenTree::Ident(tt) => Display::fmt(tt, f),
224                TokenTree::Punct(tt) => {
225                    joint = tt.spacing() == Spacing::Joint;
226                    Display::fmt(tt, f)
227                }
228                TokenTree::Literal(tt) => Display::fmt(tt, f),
229            }?;
230        }
231
232        Ok(())
233    }
234}
235
236impl Debug for TokenStream {
237    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
238        f.write_str("TokenStream ")?;
239        f.debug_list().entries(self.clone()).finish()
240    }
241}
242
243#[cfg(feature = "proc-macro")]
244impl From<proc_macro::TokenStream> for TokenStream {
245    fn from(inner: proc_macro::TokenStream) -> Self {
246        TokenStream::from_str_unchecked(&inner.to_string())
247    }
248}
249
250#[cfg(feature = "proc-macro")]
251impl From<TokenStream> for proc_macro::TokenStream {
252    fn from(inner: TokenStream) -> Self {
253        proc_macro::TokenStream::from_str_unchecked(&inner.to_string())
254    }
255}
256
257impl From<TokenTree> for TokenStream {
258    fn from(tree: TokenTree) -> Self {
259        let mut stream = RcVecBuilder::new();
260        push_token_from_proc_macro(stream.as_mut(), tree);
261        TokenStream {
262            inner: stream.build(),
263        }
264    }
265}
266
267impl FromIterator<TokenTree> for TokenStream {
268    fn from_iter<I: IntoIterator<Item = TokenTree>>(tokens: I) -> Self {
269        let mut stream = TokenStream::new();
270        stream.extend(tokens);
271        stream
272    }
273}
274
275impl FromIterator<TokenStream> for TokenStream {
276    fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
277        let mut v = RcVecBuilder::new();
278
279        for stream in streams {
280            v.extend(stream.take_inner());
281        }
282
283        TokenStream { inner: v.build() }
284    }
285}
286
287impl Extend<TokenTree> for TokenStream {
288    fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, tokens: I) {
289        let mut vec = self.inner.make_mut();
290        tokens
291            .into_iter()
292            .for_each(|token| push_token_from_proc_macro(vec.as_mut(), token));
293    }
294}
295
296impl Extend<TokenStream> for TokenStream {
297    fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
298        self.inner.make_mut().extend(streams.into_iter().flatten());
299    }
300}
301
302pub(crate) type TokenTreeIter = RcVecIntoIter<TokenTree>;
303
304impl IntoIterator for TokenStream {
305    type Item = TokenTree;
306    type IntoIter = TokenTreeIter;
307
308    fn into_iter(self) -> TokenTreeIter {
309        self.take_inner().into_iter()
310    }
311}
312
313#[cfg(all(span_locations, not(fuzzing)))]
314thread_local! {
315    static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
316        // Start with a single dummy file which all call_site() and def_site()
317        // spans reference.
318        files: vec![FileInfo {
319            source_text: String::new(),
320            span: Span { lo: 0, hi: 0 },
321            lines: vec![0],
322            char_index_to_byte_offset: BTreeMap::new(),
323        }],
324    });
325}
326
327#[cfg(span_locations)]
328pub(crate) fn invalidate_current_thread_spans() {
329    #[cfg(not(fuzzing))]
330    SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1));
331}
332
333#[cfg(all(span_locations, not(fuzzing)))]
334struct FileInfo {
335    source_text: String,
336    span: Span,
337    lines: Vec<usize>,
338    char_index_to_byte_offset: BTreeMap<usize, usize>,
339}
340
341#[cfg(all(span_locations, not(fuzzing)))]
342impl FileInfo {
343    fn offset_line_column(&self, offset: usize) -> LineColumn {
344        assert!(self.span_within(Span {
345            lo: offset as u32,
346            hi: offset as u32,
347        }));
348        let offset = offset - self.span.lo as usize;
349        match self.lines.binary_search(&offset) {
350            Ok(found) => LineColumn {
351                line: found + 1,
352                column: 0,
353            },
354            Err(idx) => LineColumn {
355                line: idx,
356                column: offset - self.lines[idx - 1],
357            },
358        }
359    }
360
361    fn span_within(&self, span: Span) -> bool {
362        span.lo >= self.span.lo && span.hi <= self.span.hi
363    }
364
365    fn byte_range(&mut self, span: Span) -> Range<usize> {
366        let lo_char = (span.lo - self.span.lo) as usize;
367
368        // Look up offset of the largest already-computed char index that is
369        // less than or equal to the current requested one. We resume counting
370        // chars from that point.
371        let (&last_char_index, &last_byte_offset) = self
372            .char_index_to_byte_offset
373            .range(..=lo_char)
374            .next_back()
375            .unwrap_or((&0, &0));
376
377        let lo_byte = if last_char_index == lo_char {
378            last_byte_offset
379        } else {
380            let total_byte_offset = match self.source_text[last_byte_offset..]
381                .char_indices()
382                .nth(lo_char - last_char_index)
383            {
384                Some((additional_offset, _ch)) => last_byte_offset + additional_offset,
385                None => self.source_text.len(),
386            };
387            self.char_index_to_byte_offset
388                .insert(lo_char, total_byte_offset);
389            total_byte_offset
390        };
391
392        let trunc_lo = &self.source_text[lo_byte..];
393        let char_len = (span.hi - span.lo) as usize;
394        lo_byte..match trunc_lo.char_indices().nth(char_len) {
395            Some((offset, _ch)) => lo_byte + offset,
396            None => self.source_text.len(),
397        }
398    }
399
400    fn source_text(&mut self, span: Span) -> String {
401        let byte_range = self.byte_range(span);
402        self.source_text[byte_range].to_owned()
403    }
404}
405
406/// Computes the offsets of each line in the given source string
407/// and the total number of characters
408#[cfg(all(span_locations, not(fuzzing)))]
409fn lines_offsets(s: &str) -> (usize, Vec<usize>) {
410    let mut lines = vec![0];
411    let mut total = 0;
412
413    for ch in s.chars() {
414        total += 1;
415        if ch == '\n' {
416            lines.push(total);
417        }
418    }
419
420    (total, lines)
421}
422
423#[cfg(all(span_locations, not(fuzzing)))]
424struct SourceMap {
425    files: Vec<FileInfo>,
426}
427
428#[cfg(all(span_locations, not(fuzzing)))]
429impl SourceMap {
430    fn next_start_pos(&self) -> u32 {
431        // Add 1 so there's always space between files.
432        //
433        // We'll always have at least 1 file, as we initialize our files list
434        // with a dummy file.
435        self.files.last().unwrap().span.hi + 1
436    }
437
438    fn add_file(&mut self, src: &str) -> Span {
439        let (len, lines) = lines_offsets(src);
440        let lo = self.next_start_pos();
441        let span = Span {
442            lo,
443            hi: lo + (len as u32),
444        };
445
446        self.files.push(FileInfo {
447            source_text: src.to_owned(),
448            span,
449            lines,
450            // Populated lazily by source_text().
451            char_index_to_byte_offset: BTreeMap::new(),
452        });
453
454        span
455    }
456
457    #[cfg(procmacro2_semver_exempt)]
458    fn filepath(&self, span: Span) -> String {
459        for (i, file) in self.files.iter().enumerate() {
460            if file.span_within(span) {
461                return if i == 0 {
462                    "<unspecified>".to_owned()
463                } else {
464                    format!("<parsed string {}>", i)
465                };
466            }
467        }
468        unreachable!("Invalid span with no related FileInfo!");
469    }
470
471    fn fileinfo(&self, span: Span) -> &FileInfo {
472        for file in &self.files {
473            if file.span_within(span) {
474                return file;
475            }
476        }
477        unreachable!("Invalid span with no related FileInfo!");
478    }
479
480    fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo {
481        for file in &mut self.files {
482            if file.span_within(span) {
483                return file;
484            }
485        }
486        unreachable!("Invalid span with no related FileInfo!");
487    }
488}
489
490#[derive(Clone, Copy, PartialEq, Eq)]
491pub(crate) struct Span {
492    #[cfg(span_locations)]
493    pub(crate) lo: u32,
494    #[cfg(span_locations)]
495    pub(crate) hi: u32,
496}
497
498impl Span {
499    #[cfg(not(span_locations))]
500    pub(crate) fn call_site() -> Self {
501        Span {}
502    }
503
504    #[cfg(span_locations)]
505    pub(crate) fn call_site() -> Self {
506        Span { lo: 0, hi: 0 }
507    }
508
509    pub(crate) fn mixed_site() -> Self {
510        Span::call_site()
511    }
512
513    #[cfg(procmacro2_semver_exempt)]
514    pub(crate) fn def_site() -> Self {
515        Span::call_site()
516    }
517
518    pub(crate) fn resolved_at(&self, _other: Span) -> Span {
519        // Stable spans consist only of line/column information, so
520        // `resolved_at` and `located_at` only select which span the
521        // caller wants line/column information from.
522        *self
523    }
524
525    pub(crate) fn located_at(&self, other: Span) -> Span {
526        other
527    }
528
529    #[cfg(span_locations)]
530    pub(crate) fn byte_range(&self) -> Range<usize> {
531        #[cfg(fuzzing)]
532        return 0..0;
533
534        #[cfg(not(fuzzing))]
535        {
536            if self.is_call_site() {
537                0..0
538            } else {
539                SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self))
540            }
541        }
542    }
543
544    #[cfg(span_locations)]
545    pub(crate) fn start(&self) -> LineColumn {
546        #[cfg(fuzzing)]
547        return LineColumn { line: 0, column: 0 };
548
549        #[cfg(not(fuzzing))]
550        SOURCE_MAP.with(|sm| {
551            let sm = sm.borrow();
552            let fi = sm.fileinfo(*self);
553            fi.offset_line_column(self.lo as usize)
554        })
555    }
556
557    #[cfg(span_locations)]
558    pub(crate) fn end(&self) -> LineColumn {
559        #[cfg(fuzzing)]
560        return LineColumn { line: 0, column: 0 };
561
562        #[cfg(not(fuzzing))]
563        SOURCE_MAP.with(|sm| {
564            let sm = sm.borrow();
565            let fi = sm.fileinfo(*self);
566            fi.offset_line_column(self.hi as usize)
567        })
568    }
569
570    #[cfg(procmacro2_semver_exempt)]
571    pub(crate) fn file(&self) -> String {
572        #[cfg(fuzzing)]
573        return "<unspecified>".to_owned();
574
575        #[cfg(not(fuzzing))]
576        SOURCE_MAP.with(|sm| {
577            let sm = sm.borrow();
578            sm.filepath(*self)
579        })
580    }
581
582    #[cfg(procmacro2_semver_exempt)]
583    pub(crate) fn local_file(&self) -> Option<PathBuf> {
584        None
585    }
586
587    #[cfg(not(span_locations))]
588    pub(crate) fn join(&self, _other: Span) -> Option<Span> {
589        Some(Span {})
590    }
591
592    #[cfg(span_locations)]
593    pub(crate) fn join(&self, other: Span) -> Option<Span> {
594        #[cfg(fuzzing)]
595        return {
596            let _ = other;
597            None
598        };
599
600        #[cfg(not(fuzzing))]
601        SOURCE_MAP.with(|sm| {
602            let sm = sm.borrow();
603            // If `other` is not within the same FileInfo as us, return None.
604            if !sm.fileinfo(*self).span_within(other) {
605                return None;
606            }
607            Some(Span {
608                lo: cmp::min(self.lo, other.lo),
609                hi: cmp::max(self.hi, other.hi),
610            })
611        })
612    }
613
614    #[cfg(not(span_locations))]
615    pub(crate) fn source_text(&self) -> Option<String> {
616        None
617    }
618
619    #[cfg(span_locations)]
620    pub(crate) fn source_text(&self) -> Option<String> {
621        #[cfg(fuzzing)]
622        return None;
623
624        #[cfg(not(fuzzing))]
625        {
626            if self.is_call_site() {
627                None
628            } else {
629                Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self)))
630            }
631        }
632    }
633
634    #[cfg(not(span_locations))]
635    pub(crate) fn first_byte(self) -> Self {
636        self
637    }
638
639    #[cfg(span_locations)]
640    pub(crate) fn first_byte(self) -> Self {
641        Span {
642            lo: self.lo,
643            hi: cmp::min(self.lo.saturating_add(1), self.hi),
644        }
645    }
646
647    #[cfg(not(span_locations))]
648    pub(crate) fn last_byte(self) -> Self {
649        self
650    }
651
652    #[cfg(span_locations)]
653    pub(crate) fn last_byte(self) -> Self {
654        Span {
655            lo: cmp::max(self.hi.saturating_sub(1), self.lo),
656            hi: self.hi,
657        }
658    }
659
660    #[cfg(span_locations)]
661    fn is_call_site(&self) -> bool {
662        self.lo == 0 && self.hi == 0
663    }
664}
665
666impl Debug for Span {
667    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
668        #[cfg(span_locations)]
669        return write!(f, "bytes({}..{})", self.lo, self.hi);
670
671        #[cfg(not(span_locations))]
672        write!(f, "Span")
673    }
674}
675
676pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
677    #[cfg(span_locations)]
678    {
679        if span.is_call_site() {
680            return;
681        }
682    }
683
684    if cfg!(span_locations) {
685        debug.field("span", &span);
686    }
687}
688
689#[derive(Clone)]
690pub(crate) struct Group {
691    delimiter: Delimiter,
692    stream: TokenStream,
693    span: Span,
694}
695
696impl Group {
697    pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
698        Group {
699            delimiter,
700            stream,
701            span: Span::call_site(),
702        }
703    }
704
705    pub(crate) fn delimiter(&self) -> Delimiter {
706        self.delimiter
707    }
708
709    pub(crate) fn stream(&self) -> TokenStream {
710        self.stream.clone()
711    }
712
713    pub(crate) fn span(&self) -> Span {
714        self.span
715    }
716
717    pub(crate) fn span_open(&self) -> Span {
718        self.span.first_byte()
719    }
720
721    pub(crate) fn span_close(&self) -> Span {
722        self.span.last_byte()
723    }
724
725    pub(crate) fn set_span(&mut self, span: Span) {
726        self.span = span;
727    }
728}
729
730impl Display for Group {
731    // We attempt to match libproc_macro's formatting.
732    // Empty parens: ()
733    // Nonempty parens: (...)
734    // Empty brackets: []
735    // Nonempty brackets: [...]
736    // Empty braces: { }
737    // Nonempty braces: { ... }
738    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
739        let (open, close) = match self.delimiter {
740            Delimiter::Parenthesis => ("(", ")"),
741            Delimiter::Brace => ("{ ", "}"),
742            Delimiter::Bracket => ("[", "]"),
743            Delimiter::None => ("", ""),
744        };
745
746        f.write_str(open)?;
747        Display::fmt(&self.stream, f)?;
748        if self.delimiter == Delimiter::Brace && !self.stream.inner.is_empty() {
749            f.write_str(" ")?;
750        }
751        f.write_str(close)?;
752
753        Ok(())
754    }
755}
756
757impl Debug for Group {
758    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
759        let mut debug = fmt.debug_struct("Group");
760        debug.field("delimiter", &self.delimiter);
761        debug.field("stream", &self.stream);
762        debug_span_field_if_nontrivial(&mut debug, self.span);
763        debug.finish()
764    }
765}
766
767#[derive(Clone)]
768pub(crate) struct Ident {
769    sym: Box<str>,
770    span: Span,
771    raw: bool,
772}
773
774impl Ident {
775    #[track_caller]
776    pub(crate) fn new_checked(string: &str, span: Span) -> Self {
777        validate_ident(string);
778        Ident::new_unchecked(string, span)
779    }
780
781    pub(crate) fn new_unchecked(string: &str, span: Span) -> Self {
782        Ident {
783            sym: Box::from(string),
784            span,
785            raw: false,
786        }
787    }
788
789    #[track_caller]
790    pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self {
791        validate_ident_raw(string);
792        Ident::new_raw_unchecked(string, span)
793    }
794
795    pub(crate) fn new_raw_unchecked(string: &str, span: Span) -> Self {
796        Ident {
797            sym: Box::from(string),
798            span,
799            raw: true,
800        }
801    }
802
803    pub(crate) fn span(&self) -> Span {
804        self.span
805    }
806
807    pub(crate) fn set_span(&mut self, span: Span) {
808        self.span = span;
809    }
810}
811
812pub(crate) fn is_ident_start(c: char) -> bool {
813    c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
814}
815
816pub(crate) fn is_ident_continue(c: char) -> bool {
817    unicode_xid::UnicodeXID::is_xid_continue(c)
818}
819
820#[track_caller]
821fn validate_ident(string: &str) {
822    if string.is_empty() {
823        panic!("Ident is not allowed to be empty; use Option<Ident>");
824    }
825
826    if string.bytes().all(|digit| b'0' <= digit && digit <= b'9') {
827        panic!("Ident cannot be a number; use Literal instead");
828    }
829
830    fn ident_ok(string: &str) -> bool {
831        let mut chars = string.chars();
832        let first = chars.next().unwrap();
833        if !is_ident_start(first) {
834            return false;
835        }
836        for ch in chars {
837            if !is_ident_continue(ch) {
838                return false;
839            }
840        }
841        true
842    }
843
844    if !ident_ok(string) {
845        panic!("{:?} is not a valid Ident", string);
846    }
847}
848
849#[track_caller]
850fn validate_ident_raw(string: &str) {
851    validate_ident(string);
852
853    match string {
854        "_" | "super" | "self" | "Self" | "crate" => {
855            panic!("`r#{}` cannot be a raw identifier", string);
856        }
857        _ => {}
858    }
859}
860
861impl PartialEq for Ident {
862    fn eq(&self, other: &Ident) -> bool {
863        self.sym == other.sym && self.raw == other.raw
864    }
865}
866
867impl<T> PartialEq<T> for Ident
868where
869    T: ?Sized + AsRef<str>,
870{
871    fn eq(&self, other: &T) -> bool {
872        let other = other.as_ref();
873        if self.raw {
874            other.starts_with("r#") && *self.sym == other[2..]
875        } else {
876            *self.sym == *other
877        }
878    }
879}
880
881impl Display for Ident {
882    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
883        if self.raw {
884            f.write_str("r#")?;
885        }
886        Display::fmt(&self.sym, f)
887    }
888}
889
890#[allow(clippy::missing_fields_in_debug)]
891impl Debug for Ident {
892    // Ident(proc_macro), Ident(r#union)
893    #[cfg(not(span_locations))]
894    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
895        let mut debug = f.debug_tuple("Ident");
896        debug.field(&format_args!("{}", self));
897        debug.finish()
898    }
899
900    // Ident {
901    //     sym: proc_macro,
902    //     span: bytes(128..138)
903    // }
904    #[cfg(span_locations)]
905    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
906        let mut debug = f.debug_struct("Ident");
907        debug.field("sym", &format_args!("{}", self));
908        debug_span_field_if_nontrivial(&mut debug, self.span);
909        debug.finish()
910    }
911}
912
913#[derive(Clone)]
914pub(crate) struct Literal {
915    pub(crate) repr: String,
916    span: Span,
917}
918
919macro_rules! suffixed_numbers {
920    ($($name:ident => $kind:ident,)*) => ($(
921        pub(crate) fn $name(n: $kind) -> Literal {
922            Literal::_new(format!(concat!("{}", stringify!($kind)), n))
923        }
924    )*)
925}
926
927macro_rules! unsuffixed_numbers {
928    ($($name:ident => $kind:ident,)*) => ($(
929        pub(crate) fn $name(n: $kind) -> Literal {
930            Literal::_new(n.to_string())
931        }
932    )*)
933}
934
935impl Literal {
936    pub(crate) fn _new(repr: String) -> Self {
937        Literal {
938            repr,
939            span: Span::call_site(),
940        }
941    }
942
943    pub(crate) fn from_str_checked(repr: &str) -> Result<Self, LexError> {
944        let mut cursor = get_cursor(repr);
945        #[cfg(span_locations)]
946        let lo = cursor.off;
947
948        let negative = cursor.starts_with_char('-');
949        if negative {
950            cursor = cursor.advance(1);
951            if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) {
952                return Err(LexError::call_site());
953            }
954        }
955
956        if let Ok((rest, mut literal)) = parse::literal(cursor) {
957            if rest.is_empty() {
958                if negative {
959                    literal.repr.insert(0, '-');
960                }
961                literal.span = Span {
962                    #[cfg(span_locations)]
963                    lo,
964                    #[cfg(span_locations)]
965                    hi: rest.off,
966                };
967                return Ok(literal);
968            }
969        }
970        Err(LexError::call_site())
971    }
972
973    pub(crate) fn from_str_unchecked(repr: &str) -> Self {
974        Self::from_str_checked(repr).unwrap()
975    }
976
977    suffixed_numbers! {
978        u8_suffixed => u8,
979        u16_suffixed => u16,
980        u32_suffixed => u32,
981        u64_suffixed => u64,
982        u128_suffixed => u128,
983        usize_suffixed => usize,
984        i8_suffixed => i8,
985        i16_suffixed => i16,
986        i32_suffixed => i32,
987        i64_suffixed => i64,
988        i128_suffixed => i128,
989        isize_suffixed => isize,
990
991        f32_suffixed => f32,
992        f64_suffixed => f64,
993    }
994
995    unsuffixed_numbers! {
996        u8_unsuffixed => u8,
997        u16_unsuffixed => u16,
998        u32_unsuffixed => u32,
999        u64_unsuffixed => u64,
1000        u128_unsuffixed => u128,
1001        usize_unsuffixed => usize,
1002        i8_unsuffixed => i8,
1003        i16_unsuffixed => i16,
1004        i32_unsuffixed => i32,
1005        i64_unsuffixed => i64,
1006        i128_unsuffixed => i128,
1007        isize_unsuffixed => isize,
1008    }
1009
1010    pub(crate) fn f32_unsuffixed(f: f32) -> Literal {
1011        let mut s = f.to_string();
1012        if !s.contains('.') {
1013            s.push_str(".0");
1014        }
1015        Literal::_new(s)
1016    }
1017
1018    pub(crate) fn f64_unsuffixed(f: f64) -> Literal {
1019        let mut s = f.to_string();
1020        if !s.contains('.') {
1021            s.push_str(".0");
1022        }
1023        Literal::_new(s)
1024    }
1025
1026    pub(crate) fn string(string: &str) -> Literal {
1027        let mut repr = String::with_capacity(string.len() + 2);
1028        repr.push('"');
1029        escape_utf8(string, &mut repr);
1030        repr.push('"');
1031        Literal::_new(repr)
1032    }
1033
1034    pub(crate) fn character(ch: char) -> Literal {
1035        let mut repr = String::new();
1036        repr.push('\'');
1037        if ch == '"' {
1038            // escape_debug turns this into '\"' which is unnecessary.
1039            repr.push(ch);
1040        } else {
1041            repr.extend(ch.escape_debug());
1042        }
1043        repr.push('\'');
1044        Literal::_new(repr)
1045    }
1046
1047    pub(crate) fn byte_character(byte: u8) -> Literal {
1048        let mut repr = "b'".to_string();
1049        #[allow(clippy::match_overlapping_arm)]
1050        match byte {
1051            b'\0' => repr.push_str(r"\0"),
1052            b'\t' => repr.push_str(r"\t"),
1053            b'\n' => repr.push_str(r"\n"),
1054            b'\r' => repr.push_str(r"\r"),
1055            b'\'' => repr.push_str(r"\'"),
1056            b'\\' => repr.push_str(r"\\"),
1057            b'\x20'..=b'\x7E' => repr.push(byte as char),
1058            _ => {
1059                let _ = write!(repr, r"\x{:02X}", byte);
1060            }
1061        }
1062        repr.push('\'');
1063        Literal::_new(repr)
1064    }
1065
1066    pub(crate) fn byte_string(bytes: &[u8]) -> Literal {
1067        let mut repr = "b\"".to_string();
1068        let mut bytes = bytes.iter();
1069        while let Some(&b) = bytes.next() {
1070            #[allow(clippy::match_overlapping_arm)]
1071            match b {
1072                b'\0' => repr.push_str(match bytes.as_slice().first() {
1073                    // circumvent clippy::octal_escapes lint
1074                    Some(b'0'..=b'7') => r"\x00",
1075                    _ => r"\0",
1076                }),
1077                b'\t' => repr.push_str(r"\t"),
1078                b'\n' => repr.push_str(r"\n"),
1079                b'\r' => repr.push_str(r"\r"),
1080                b'"' => repr.push_str("\\\""),
1081                b'\\' => repr.push_str(r"\\"),
1082                b'\x20'..=b'\x7E' => repr.push(b as char),
1083                _ => {
1084                    let _ = write!(repr, r"\x{:02X}", b);
1085                }
1086            }
1087        }
1088        repr.push('"');
1089        Literal::_new(repr)
1090    }
1091
1092    pub(crate) fn c_string(string: &CStr) -> Literal {
1093        let mut repr = "c\"".to_string();
1094        let mut bytes = string.to_bytes();
1095        while !bytes.is_empty() {
1096            let (valid, invalid) = match str::from_utf8(bytes) {
1097                Ok(all_valid) => {
1098                    bytes = b"";
1099                    (all_valid, bytes)
1100                }
1101                Err(utf8_error) => {
1102                    let (valid, rest) = bytes.split_at(utf8_error.valid_up_to());
1103                    let valid = str::from_utf8(valid).unwrap();
1104                    let invalid = utf8_error
1105                        .error_len()
1106                        .map_or(rest, |error_len| &rest[..error_len]);
1107                    bytes = &bytes[valid.len() + invalid.len()..];
1108                    (valid, invalid)
1109                }
1110            };
1111            escape_utf8(valid, &mut repr);
1112            for &byte in invalid {
1113                let _ = write!(repr, r"\x{:02X}", byte);
1114            }
1115        }
1116        repr.push('"');
1117        Literal::_new(repr)
1118    }
1119
1120    pub(crate) fn span(&self) -> Span {
1121        self.span
1122    }
1123
1124    pub(crate) fn set_span(&mut self, span: Span) {
1125        self.span = span;
1126    }
1127
1128    pub(crate) fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
1129        #[cfg(not(span_locations))]
1130        {
1131            let _ = range;
1132            None
1133        }
1134
1135        #[cfg(span_locations)]
1136        {
1137            use core::ops::Bound;
1138
1139            let lo = match range.start_bound() {
1140                Bound::Included(start) => {
1141                    let start = u32::try_from(*start).ok()?;
1142                    self.span.lo.checked_add(start)?
1143                }
1144                Bound::Excluded(start) => {
1145                    let start = u32::try_from(*start).ok()?;
1146                    self.span.lo.checked_add(start)?.checked_add(1)?
1147                }
1148                Bound::Unbounded => self.span.lo,
1149            };
1150            let hi = match range.end_bound() {
1151                Bound::Included(end) => {
1152                    let end = u32::try_from(*end).ok()?;
1153                    self.span.lo.checked_add(end)?.checked_add(1)?
1154                }
1155                Bound::Excluded(end) => {
1156                    let end = u32::try_from(*end).ok()?;
1157                    self.span.lo.checked_add(end)?
1158                }
1159                Bound::Unbounded => self.span.hi,
1160            };
1161            if lo <= hi && hi <= self.span.hi {
1162                Some(Span { lo, hi })
1163            } else {
1164                None
1165            }
1166        }
1167    }
1168}
1169
1170impl Display for Literal {
1171    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1172        Display::fmt(&self.repr, f)
1173    }
1174}
1175
1176impl Debug for Literal {
1177    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1178        let mut debug = fmt.debug_struct("Literal");
1179        debug.field("lit", &format_args!("{}", self.repr));
1180        debug_span_field_if_nontrivial(&mut debug, self.span);
1181        debug.finish()
1182    }
1183}
1184
1185fn escape_utf8(string: &str, repr: &mut String) {
1186    let mut chars = string.chars();
1187    while let Some(ch) = chars.next() {
1188        if ch == '\0' {
1189            repr.push_str(
1190                if chars
1191                    .as_str()
1192                    .starts_with(|next| '0' <= next && next <= '7')
1193                {
1194                    // circumvent clippy::octal_escapes lint
1195                    r"\x00"
1196                } else {
1197                    r"\0"
1198                },
1199            );
1200        } else if ch == '\'' {
1201            // escape_debug turns this into "\'" which is unnecessary.
1202            repr.push(ch);
1203        } else {
1204            repr.extend(ch.escape_debug());
1205        }
1206    }
1207}
1208
1209#[cfg(feature = "proc-macro")]
1210pub(crate) trait FromStr2: FromStr<Err = proc_macro::LexError> {
1211    #[cfg(wrap_proc_macro)]
1212    fn valid(src: &str) -> bool;
1213
1214    #[cfg(wrap_proc_macro)]
1215    fn from_str_checked(src: &str) -> Result<Self, imp::LexError> {
1216        // Validate using fallback parser, because rustc is incapable of
1217        // returning a recoverable Err for certain invalid token streams, and
1218        // will instead permanently poison the compilation.
1219        if !Self::valid(src) {
1220            return Err(imp::LexError::CompilerPanic);
1221        }
1222
1223        // Catch panic to work around https://github.com/rust-lang/rust/issues/58736.
1224        match panic::catch_unwind(|| Self::from_str(src)) {
1225            Ok(Ok(ok)) => Ok(ok),
1226            Ok(Err(lex)) => Err(imp::LexError::Compiler(lex)),
1227            Err(_panic) => Err(imp::LexError::CompilerPanic),
1228        }
1229    }
1230
1231    fn from_str_unchecked(src: &str) -> Self {
1232        Self::from_str(src).unwrap()
1233    }
1234}
1235
1236#[cfg(feature = "proc-macro")]
1237impl FromStr2 for proc_macro::TokenStream {
1238    #[cfg(wrap_proc_macro)]
1239    fn valid(src: &str) -> bool {
1240        TokenStream::from_str_checked(src).is_ok()
1241    }
1242}
1243
1244#[cfg(feature = "proc-macro")]
1245impl FromStr2 for proc_macro::Literal {
1246    #[cfg(wrap_proc_macro)]
1247    fn valid(src: &str) -> bool {
1248        Literal::from_str_checked(src).is_ok()
1249    }
1250}