pub use BinOpToken::*;
pub use Nonterminal::*;
pub use DelimToken::*;
pub use Lit::*;
pub use Token::*;
use crate::ast::{self};
use crate::parse::ParseSess;
use crate::print::pprust;
use crate::ptr::P;
use crate::symbol::kw;
use crate::syntax::parse::parse_stream_from_source_str;
use crate::tokenstream::{self, DelimSpan, TokenStream, TokenTree};
use syntax_pos::symbol::{self, Symbol};
use syntax_pos::{self, Span, FileName};
use log::info;
use std::fmt;
use std::mem;
#[cfg(target_arch = "x86_64")]
use rustc_data_structures::static_assert_size;
use rustc_data_structures::sync::Lrc;
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
pub enum BinOpToken {
Plus,
Minus,
Star,
Slash,
Percent,
Caret,
And,
Or,
Shl,
Shr,
}
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
pub enum DelimToken {
Paren,
Bracket,
Brace,
NoDelim,
}
impl DelimToken {
pub fn len(self) -> usize {
if self == NoDelim { 0 } else { 1 }
}
pub fn is_empty(self) -> bool {
self == NoDelim
}
}
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
pub enum Lit {
Bool(ast::Name),
Byte(ast::Name),
Char(ast::Name),
Err(ast::Name),
Integer(ast::Name),
Float(ast::Name),
Str_(ast::Name),
StrRaw(ast::Name, u16),
ByteStr(ast::Name),
ByteStrRaw(ast::Name, u16),
}
#[cfg(target_arch = "x86_64")]
static_assert_size!(Lit, 8);
impl Lit {
crate fn literal_name(&self) -> &'static str {
match *self {
Bool(_) => panic!("literal token contains `Lit::Bool`"),
Byte(_) => "byte literal",
Char(_) => "char literal",
Err(_) => "invalid literal",
Integer(_) => "integer literal",
Float(_) => "float literal",
Str_(_) | StrRaw(..) => "string literal",
ByteStr(_) | ByteStrRaw(..) => "byte string literal"
}
}
crate fn may_have_suffix(&self) -> bool {
match *self {
Integer(..) | Float(..) => true,
_ => false,
}
}
fn probably_equal_for_proc_macro(&self, other: &Lit) -> bool {
mem::discriminant(self) == mem::discriminant(other)
}
}
pub(crate) fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool {
let ident_token: Token = Ident(ident, is_raw);
!ident_token.is_reserved_ident() ||
ident_token.is_path_segment_keyword() ||
[
kw::Async,
kw::Await,
kw::Do,
kw::Box,
kw::Break,
kw::Continue,
kw::False,
kw::For,
kw::If,
kw::Loop,
kw::Match,
kw::Move,
kw::Return,
kw::True,
kw::Unsafe,
kw::While,
kw::Yield,
kw::Static,
].contains(&ident.name)
}
fn ident_can_begin_type(ident: ast::Ident, is_raw: bool) -> bool {
let ident_token: Token = Ident(ident, is_raw);
!ident_token.is_reserved_ident() ||
ident_token.is_path_segment_keyword() ||
[
kw::Underscore,
kw::For,
kw::Impl,
kw::Fn,
kw::Unsafe,
kw::Extern,
kw::Typeof,
kw::Dyn,
].contains(&ident.name)
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Debug)]
pub enum Token {
Eq,
Lt,
Le,
EqEq,
Ne,
Ge,
Gt,
AndAnd,
OrOr,
Not,
Tilde,
BinOp(BinOpToken),
BinOpEq(BinOpToken),
At,
Dot,
DotDot,
DotDotDot,
DotDotEq,
Comma,
Semi,
Colon,
ModSep,
RArrow,
LArrow,
FatArrow,
Pound,
Dollar,
Question,
SingleQuote,
OpenDelim(DelimToken),
CloseDelim(DelimToken),
Literal(Lit, Option<ast::Name>),
Ident(ast::Ident, bool),
Lifetime(ast::Ident),
Interpolated(Lrc<Nonterminal>),
DocComment(ast::Name),
Whitespace,
Comment,
Shebang(ast::Name),
Eof,
}
#[cfg(target_arch = "x86_64")]
static_assert_size!(Token, 16);
impl Token {
pub fn from_ast_ident(ident: ast::Ident) -> Token {
Ident(ident, ident.is_raw_guess())
}
crate fn is_like_plus(&self) -> bool {
match *self {
BinOp(Plus) | BinOpEq(Plus) => true,
_ => false,
}
}
crate fn can_begin_expr(&self) -> bool {
match *self {
Ident(ident, is_raw) =>
ident_can_begin_expr(ident, is_raw),
OpenDelim(..) |
Literal(..) |
Not |
BinOp(Minus) |
BinOp(Star) |
BinOp(Or) | OrOr |
BinOp(And) |
AndAnd |
DotDot | DotDotDot | DotDotEq |
Lt | BinOp(Shl) |
ModSep |
Lifetime(..) |
Pound => true,
Interpolated(ref nt) => match **nt {
NtLiteral(..) |
NtIdent(..) |
NtExpr(..) |
NtBlock(..) |
NtPath(..) |
NtLifetime(..) => true,
_ => false,
},
_ => false,
}
}
crate fn can_begin_type(&self) -> bool {
match *self {
Ident(ident, is_raw) =>
ident_can_begin_type(ident, is_raw),
OpenDelim(Paren) |
OpenDelim(Bracket) |
Not |
BinOp(Star) |
BinOp(And) |
AndAnd |
Question |
Lifetime(..) |
Lt | BinOp(Shl) |
ModSep => true,
Interpolated(ref nt) => match **nt {
NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true,
_ => false,
},
_ => false,
}
}
pub fn can_begin_const_arg(&self) -> bool {
match self {
OpenDelim(Brace) => true,
Interpolated(ref nt) => match **nt {
NtExpr(..) => true,
NtBlock(..) => true,
NtLiteral(..) => true,
_ => false,
}
_ => self.can_begin_literal_or_bool(),
}
}
crate fn can_begin_bound(&self) -> bool {
self.is_path_start() || self.is_lifetime() || self.is_keyword(kw::For) ||
self == &Question || self == &OpenDelim(Paren)
}
crate fn is_lit(&self) -> bool {
match *self {
Literal(..) => true,
_ => false,
}
}
crate fn can_begin_literal_or_bool(&self) -> bool {
match *self {
Literal(..) => true,
BinOp(Minus) => true,
Ident(ident, false) if ident.name == kw::True => true,
Ident(ident, false) if ident.name == kw::False => true,
Interpolated(ref nt) => match **nt {
NtLiteral(..) => true,
_ => false,
},
_ => false,
}
}
pub fn ident(&self) -> Option<(ast::Ident, bool)> {
match *self {
Ident(ident, is_raw) => Some((ident, is_raw)),
Interpolated(ref nt) => match **nt {
NtIdent(ident, is_raw) => Some((ident, is_raw)),
_ => None,
},
_ => None,
}
}
pub fn lifetime(&self) -> Option<ast::Ident> {
match *self {
Lifetime(ident) => Some(ident),
Interpolated(ref nt) => match **nt {
NtLifetime(ident) => Some(ident),
_ => None,
},
_ => None,
}
}
pub fn is_ident(&self) -> bool {
self.ident().is_some()
}
crate fn is_lifetime(&self) -> bool {
self.lifetime().is_some()
}
crate fn is_ident_named(&self, name: &str) -> bool {
match self.ident() {
Some((ident, _)) => ident.as_str() == name,
None => false
}
}
fn is_path(&self) -> bool {
if let Interpolated(ref nt) = *self {
if let NtPath(..) = **nt {
return true;
}
}
false
}
crate fn is_mutability(&self) -> bool {
self.is_keyword(kw::Mut) ||
self.is_keyword(kw::Const)
}
crate fn is_qpath_start(&self) -> bool {
self == &Lt || self == &BinOp(Shl)
}
crate fn is_path_start(&self) -> bool {
self == &ModSep || self.is_qpath_start() || self.is_path() ||
self.is_path_segment_keyword() || self.is_ident() && !self.is_reserved_ident()
}
pub fn is_keyword(&self, kw: Symbol) -> bool {
self.ident().map(|(ident, is_raw)| ident.name == kw && !is_raw).unwrap_or(false)
}
pub fn is_path_segment_keyword(&self) -> bool {
match self.ident() {
Some((id, false)) => id.is_path_segment_keyword(),
_ => false,
}
}
pub fn is_special_ident(&self) -> bool {
match self.ident() {
Some((id, false)) => id.is_special(),
_ => false,
}
}
crate fn is_used_keyword(&self) -> bool {
match self.ident() {
Some((id, false)) => id.is_used_keyword(),
_ => false,
}
}
crate fn is_unused_keyword(&self) -> bool {
match self.ident() {
Some((id, false)) => id.is_unused_keyword(),
_ => false,
}
}
pub fn is_reserved_ident(&self) -> bool {
match self.ident() {
Some((id, false)) => id.is_reserved(),
_ => false,
}
}
crate fn glue(self, joint: Token) -> Option<Token> {
Some(match self {
Eq => match joint {
Eq => EqEq,
Gt => FatArrow,
_ => return None,
},
Lt => match joint {
Eq => Le,
Lt => BinOp(Shl),
Le => BinOpEq(Shl),
BinOp(Minus) => LArrow,
_ => return None,
},
Gt => match joint {
Eq => Ge,
Gt => BinOp(Shr),
Ge => BinOpEq(Shr),
_ => return None,
},
Not => match joint {
Eq => Ne,
_ => return None,
},
BinOp(op) => match joint {
Eq => BinOpEq(op),
BinOp(And) if op == And => AndAnd,
BinOp(Or) if op == Or => OrOr,
Gt if op == Minus => RArrow,
_ => return None,
},
Dot => match joint {
Dot => DotDot,
DotDot => DotDotDot,
_ => return None,
},
DotDot => match joint {
Dot => DotDotDot,
Eq => DotDotEq,
_ => return None,
},
Colon => match joint {
Colon => ModSep,
_ => return None,
},
SingleQuote => match joint {
Ident(ident, false) => {
let name = Symbol::intern(&format!("'{}", ident));
Lifetime(symbol::Ident {
name,
span: ident.span,
})
}
_ => return None,
},
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot |
DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar |
Question | OpenDelim(..) | CloseDelim(..) |
Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) |
Whitespace | Comment | Shebang(..) | Eof => return None,
})
}
crate fn similar_tokens(&self) -> Option<Vec<Token>> {
match *self {
Comma => Some(vec![Dot, Lt, Semi]),
Semi => Some(vec![Colon, Comma]),
_ => None
}
}
crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool {
if mem::discriminant(self) != mem::discriminant(other) {
return false
}
match (self, other) {
(&Eq, &Eq) |
(&Lt, &Lt) |
(&Le, &Le) |
(&EqEq, &EqEq) |
(&Ne, &Ne) |
(&Ge, &Ge) |
(&Gt, &Gt) |
(&AndAnd, &AndAnd) |
(&OrOr, &OrOr) |
(&Not, &Not) |
(&Tilde, &Tilde) |
(&At, &At) |
(&Dot, &Dot) |
(&DotDot, &DotDot) |
(&DotDotDot, &DotDotDot) |
(&DotDotEq, &DotDotEq) |
(&Comma, &Comma) |
(&Semi, &Semi) |
(&Colon, &Colon) |
(&ModSep, &ModSep) |
(&RArrow, &RArrow) |
(&LArrow, &LArrow) |
(&FatArrow, &FatArrow) |
(&Pound, &Pound) |
(&Dollar, &Dollar) |
(&Question, &Question) |
(&Whitespace, &Whitespace) |
(&Comment, &Comment) |
(&Eof, &Eof) => true,
(&BinOp(a), &BinOp(b)) |
(&BinOpEq(a), &BinOpEq(b)) => a == b,
(&OpenDelim(a), &OpenDelim(b)) |
(&CloseDelim(a), &CloseDelim(b)) => a == b,
(&DocComment(a), &DocComment(b)) |
(&Shebang(a), &Shebang(b)) => a == b,
(&Lifetime(a), &Lifetime(b)) => a.name == b.name,
(&Ident(a, b), &Ident(c, d)) => b == d && (a.name == c.name ||
a.name == kw::DollarCrate ||
c.name == kw::DollarCrate),
(&Literal(ref a, b), &Literal(ref c, d)) => {
b == d && a.probably_equal_for_proc_macro(c)
}
(&Interpolated(_), &Interpolated(_)) => false,
_ => panic!("forgot to add a token?"),
}
}
}
#[derive(Clone, RustcEncodable, RustcDecodable)]
pub enum Nonterminal {
NtItem(P<ast::Item>),
NtBlock(P<ast::Block>),
NtStmt(ast::Stmt),
NtPat(P<ast::Pat>),
NtExpr(P<ast::Expr>),
NtTy(P<ast::Ty>),
NtIdent(ast::Ident, bool),
NtLifetime(ast::Ident),
NtLiteral(P<ast::Expr>),
NtMeta(ast::MetaItem),
NtPath(ast::Path),
NtVis(ast::Visibility),
NtTT(TokenTree),
NtTraitItem(ast::TraitItem),
NtImplItem(ast::ImplItem),
NtForeignItem(ast::ForeignItem),
}
impl PartialEq for Nonterminal {
fn eq(&self, rhs: &Self) -> bool {
match (self, rhs) {
(NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) =>
ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs,
(NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
(NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs,
_ => false,
}
}
}
impl fmt::Debug for Nonterminal {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
NtItem(..) => f.pad("NtItem(..)"),
NtBlock(..) => f.pad("NtBlock(..)"),
NtStmt(..) => f.pad("NtStmt(..)"),
NtPat(..) => f.pad("NtPat(..)"),
NtExpr(..) => f.pad("NtExpr(..)"),
NtTy(..) => f.pad("NtTy(..)"),
NtIdent(..) => f.pad("NtIdent(..)"),
NtLiteral(..) => f.pad("NtLiteral(..)"),
NtMeta(..) => f.pad("NtMeta(..)"),
NtPath(..) => f.pad("NtPath(..)"),
NtTT(..) => f.pad("NtTT(..)"),
NtImplItem(..) => f.pad("NtImplItem(..)"),
NtTraitItem(..) => f.pad("NtTraitItem(..)"),
NtForeignItem(..) => f.pad("NtForeignItem(..)"),
NtVis(..) => f.pad("NtVis(..)"),
NtLifetime(..) => f.pad("NtLifetime(..)"),
}
}
}
impl Nonterminal {
pub fn to_tokenstream(&self, sess: &ParseSess, span: Span) -> TokenStream {
let tokens = match *self {
Nonterminal::NtItem(ref item) => {
prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span)
}
Nonterminal::NtTraitItem(ref item) => {
prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span)
}
Nonterminal::NtImplItem(ref item) => {
prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span)
}
Nonterminal::NtIdent(ident, is_raw) => {
let token = Token::Ident(ident, is_raw);
Some(TokenTree::Token(ident.span, token).into())
}
Nonterminal::NtLifetime(ident) => {
let token = Token::Lifetime(ident);
Some(TokenTree::Token(ident.span, token).into())
}
Nonterminal::NtTT(ref tt) => {
Some(tt.clone().into())
}
_ => None,
};
let source = pprust::nonterminal_to_string(self);
let filename = FileName::macro_expansion_source_code(&source);
let tokens_for_real = parse_stream_from_source_str(filename, source, sess, Some(span));
if let Some(tokens) = tokens {
if tokens.probably_equal_for_proc_macro(&tokens_for_real) {
return tokens
}
info!("cached tokens found, but they're not \"probably equal\", \
going with stringified version");
}
return tokens_for_real
}
}
crate fn is_op(tok: &Token) -> bool {
match *tok {
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) |
Ident(..) | Lifetime(..) | Interpolated(..) |
Whitespace | Comment | Shebang(..) | Eof => false,
_ => true,
}
}
fn prepend_attrs(sess: &ParseSess,
attrs: &[ast::Attribute],
tokens: Option<&tokenstream::TokenStream>,
span: syntax_pos::Span)
-> Option<tokenstream::TokenStream>
{
let tokens = tokens?;
if attrs.len() == 0 {
return Some(tokens.clone())
}
let mut builder = tokenstream::TokenStreamBuilder::new();
for attr in attrs {
assert_eq!(attr.style, ast::AttrStyle::Outer,
"inner attributes should prevent cached tokens from existing");
let source = pprust::attr_to_string(attr);
let macro_filename = FileName::macro_expansion_source_code(&source);
if attr.is_sugared_doc {
let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span));
builder.push(stream);
continue
}
let mut brackets = tokenstream::TokenStreamBuilder::new();
if attr.path.segments.len() == 1 && attr.path.segments[0].args.is_none() {
let ident = attr.path.segments[0].ident;
let token = Ident(ident, ident.as_str().starts_with("r#"));
brackets.push(tokenstream::TokenTree::Token(ident.span, token));
} else {
let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span));
brackets.push(stream);
}
brackets.push(attr.tokens.clone());
builder.push(tokenstream::TokenTree::Token(attr.span, Pound));
let delim_span = DelimSpan::from_single(attr.span);
builder.push(tokenstream::TokenTree::Delimited(
delim_span, DelimToken::Bracket, brackets.build().into()));
}
builder.push(tokens.clone());
Some(builder.build())
}