Enum proc_macro2::TokenTree

pub enum TokenTree {
    Group(Group),
    Ident(Ident),
    Punct(Punct),
    Literal(Literal),
}

A single token or a delimited sequence of token trees (e.g. [1, (), ..]).

Variants

Group(Group)

A token stream surrounded by bracket delimiters.

Ident(Ident)

An identifier.

Punct(Punct)

A single punctuation character (+, ,, $, etc.).

Literal(Literal)

A literal character ('a'), string ("hello"), number (2.3), etc.

Implementations

impl TokenTree

pub fn span(&self) -> Span

Returns the span of this tree, delegating to the span method of the contained token or a delimited stream.

pub fn set_span(&mut self, span: Span)

Configures the span for only this token.

Note that if this token is a Group then this method will not configure the span of each of the internal tokens, this will simply delegate to the set_span method of each variant.

Examples found in repository

src/parse.rs (lines 226-231)

pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
    let mut trees = TokenStreamBuilder::new();
    let mut stack = Vec::new();

    loop {
        input = skip_whitespace(input);

        if let Ok((rest, ())) = doc_comment(input, &mut trees) {
            input = rest;
            continue;
        }

        #[cfg(span_locations)]
        let lo = input.off;

        let first = match input.bytes().next() {
            Some(first) => first,
            None => match stack.last() {
                None => return Ok(trees.build()),
                #[cfg(span_locations)]
                Some((lo, _frame)) => {
                    return Err(LexError {
                        span: Span { lo: *lo, hi: *lo },
                    })
                }
                #[cfg(not(span_locations))]
                Some(_frame) => return Err(LexError { span: Span {} }),
            },
        };

        if let Some(open_delimiter) = match first {
            b'(' => Some(Delimiter::Parenthesis),
            b'[' => Some(Delimiter::Bracket),
            b'{' => Some(Delimiter::Brace),
            _ => None,
        } {
            input = input.advance(1);
            let frame = (open_delimiter, trees);
            #[cfg(span_locations)]
            let frame = (lo, frame);
            stack.push(frame);
            trees = TokenStreamBuilder::new();
        } else if let Some(close_delimiter) = match first {
            b')' => Some(Delimiter::Parenthesis),
            b']' => Some(Delimiter::Bracket),
            b'}' => Some(Delimiter::Brace),
            _ => None,
        } {
            let frame = match stack.pop() {
                Some(frame) => frame,
                None => return Err(lex_error(input)),
            };
            #[cfg(span_locations)]
            let (lo, frame) = frame;
            let (open_delimiter, outer) = frame;
            if open_delimiter != close_delimiter {
                return Err(lex_error(input));
            }
            input = input.advance(1);
            let mut g = Group::new(open_delimiter, trees.build());
            g.set_span(Span {
                #[cfg(span_locations)]
                lo,
                #[cfg(span_locations)]
                hi: input.off,
            });
            trees = outer;
            trees.push_token_from_parser(TokenTree::Group(crate::Group::_new_stable(g)));
        } else {
            let (rest, mut tt) = match leaf_token(input) {
                Ok((rest, tt)) => (rest, tt),
                Err(Reject) => return Err(lex_error(input)),
            };
            tt.set_span(crate::Span::_new_stable(Span {
                #[cfg(span_locations)]
                lo,
                #[cfg(span_locations)]
                hi: rest.off,
            }));
            trees.push_token_from_parser(tt);
            input = rest;
        }
    }
}

Trait Implementations

impl Clone for TokenTree

fn clone(&self) -> TokenTree

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenTree

Prints token tree in a form convenient for debugging.

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Display for TokenTree

Prints the token tree as a string that is supposed to be losslessly convertible back into the same token tree (modulo spans), except for possibly TokenTree::Groups with Delimiter::None delimiters and negative numeric literals.

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Extend<TokenTree> for TokenStream

fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I)

Extends a collection with the contents of an iterator.

fn extend_one(&mut self, item: A)

🔬This is a nightly-only experimental API. (extend_one)

Extends a collection with exactly one element.

fn extend_reserve(&mut self, additional: usize)

🔬This is a nightly-only experimental API. (extend_one)

Reserves capacity in a collection for the given number of additional elements.

impl From<Group> for TokenTree

fn from(g: Group) -> TokenTree

Converts to this type from the input type.

impl From<Ident> for TokenTree

fn from(g: Ident) -> TokenTree

Converts to this type from the input type.

impl From<Literal> for TokenTree

fn from(g: Literal) -> TokenTree

Converts to this type from the input type.

impl From<Punct> for TokenTree

fn from(g: Punct) -> TokenTree

Converts to this type from the input type.

impl From<TokenTree> for TokenStream

fn from(token: TokenTree) -> Self

Converts to this type from the input type.

impl FromIterator<TokenTree> for TokenStream

Collects a number of token trees into a single stream.

fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self

Creates a value from an iterator.