pub use self::BinOpToken::*;
pub use self::Nonterminal::*;
pub use self::DelimToken::*;
pub use self::Lit::*;
pub use self::Token::*;
use ast::{self};
use parse::ParseSess;
use print::pprust;
use ptr::P;
use serialize::{Decodable, Decoder, Encodable, Encoder};
use symbol::keywords;
use syntax::parse::parse_stream_from_source_str;
use syntax_pos::{self, Span, FileName};
use tokenstream::{TokenStream, TokenTree};
use tokenstream;
use std::cell::Cell;
use std::{cmp, fmt};
use rustc_data_structures::sync::Lrc;
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
pub enum BinOpToken {
Plus,
Minus,
Star,
Slash,
Percent,
Caret,
And,
Or,
Shl,
Shr,
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
pub enum DelimToken {
Paren,
Bracket,
Brace,
NoDelim,
}
impl DelimToken {
pub fn len(self) -> usize {
if self == NoDelim { 0 } else { 1 }
}
pub fn is_empty(self) -> bool {
self == NoDelim
}
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
pub enum Lit {
Byte(ast::Name),
Char(ast::Name),
Integer(ast::Name),
Float(ast::Name),
Str_(ast::Name),
StrRaw(ast::Name, usize),
ByteStr(ast::Name),
ByteStrRaw(ast::Name, usize),
}
impl Lit {
pub fn short_name(&self) -> &'static str {
match *self {
Byte(_) => "byte",
Char(_) => "char",
Integer(_) => "integer",
Float(_) => "float",
Str_(_) | StrRaw(..) => "string",
ByteStr(_) | ByteStrRaw(..) => "byte string"
}
}
}
fn ident_can_begin_expr(ident: ast::Ident) -> bool {
let ident_token: Token = Ident(ident);
!ident_token.is_reserved_ident() ||
ident_token.is_path_segment_keyword() ||
[
keywords::Do.name(),
keywords::Box.name(),
keywords::Break.name(),
keywords::Continue.name(),
keywords::False.name(),
keywords::For.name(),
keywords::If.name(),
keywords::Loop.name(),
keywords::Match.name(),
keywords::Move.name(),
keywords::Return.name(),
keywords::True.name(),
keywords::Unsafe.name(),
keywords::While.name(),
keywords::Yield.name(),
keywords::Static.name(),
].contains(&ident.name)
}
fn ident_can_begin_type(ident: ast::Ident) -> bool {
let ident_token: Token = Ident(ident);
!ident_token.is_reserved_ident() ||
ident_token.is_path_segment_keyword() ||
[
keywords::For.name(),
keywords::Impl.name(),
keywords::Fn.name(),
keywords::Unsafe.name(),
keywords::Extern.name(),
keywords::Typeof.name(),
].contains(&ident.name)
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug)]
pub enum Token {
Eq,
Lt,
Le,
EqEq,
Ne,
Ge,
Gt,
AndAnd,
OrOr,
Not,
Tilde,
BinOp(BinOpToken),
BinOpEq(BinOpToken),
At,
Dot,
DotDot,
DotDotDot,
DotDotEq,
DotEq, Comma,
Semi,
Colon,
ModSep,
RArrow,
LArrow,
FatArrow,
Pound,
Dollar,
Question,
OpenDelim(DelimToken),
CloseDelim(DelimToken),
Literal(Lit, Option<ast::Name>),
Ident(ast::Ident),
Underscore,
Lifetime(ast::Ident),
Interpolated(Lrc<(Nonterminal, LazyTokenStream)>),
DocComment(ast::Name),
Whitespace,
Comment,
Shebang(ast::Name),
Eof,
}
impl Token {
pub fn interpolated(nt: Nonterminal) -> Token {
Token::Interpolated(Lrc::new((nt, LazyTokenStream::new())))
}
pub fn is_like_gt(&self) -> bool {
match *self {
BinOp(Shr) | BinOpEq(Shr) | Gt | Ge => true,
_ => false,
}
}
pub fn can_begin_expr(&self) -> bool {
match *self {
Ident(ident) => ident_can_begin_expr(ident), OpenDelim(..) | Literal(..) | Not | BinOp(Minus) | BinOp(Star) | BinOp(Or) | OrOr | BinOp(And) | AndAnd | DotDot | DotDotDot | DotDotEq | Lt | BinOp(Shl) | ModSep | Pound => true, Interpolated(ref nt) => match nt.0 {
NtIdent(..) | NtExpr(..) | NtBlock(..) | NtPath(..) => true,
_ => false,
},
_ => false,
}
}
pub fn can_begin_type(&self) -> bool {
match *self {
Ident(ident) => ident_can_begin_type(ident), OpenDelim(Paren) | OpenDelim(Bracket) | Underscore | Not | BinOp(Star) | BinOp(And) | AndAnd | Question | Lifetime(..) | Lt | BinOp(Shl) | ModSep => true, Interpolated(ref nt) => match nt.0 {
NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true,
_ => false,
},
_ => false,
}
}
pub fn can_begin_bound(&self) -> bool {
self.is_path_start() || self.is_lifetime() || self.is_keyword(keywords::For) ||
self == &Question || self == &OpenDelim(Paren)
}
pub fn is_lit(&self) -> bool {
match *self {
Literal(..) => true,
_ => false,
}
}
pub fn ident(&self) -> Option<ast::Ident> {
match *self {
Ident(ident) => Some(ident),
Interpolated(ref nt) => match nt.0 {
NtIdent(ident) => Some(ident.node),
_ => None,
},
_ => None,
}
}
pub fn is_ident(&self) -> bool {
self.ident().is_some()
}
pub fn is_doc_comment(&self) -> bool {
match *self {
DocComment(..) => true,
_ => false,
}
}
pub fn is_interpolated(&self) -> bool {
match *self {
Interpolated(..) => true,
_ => false,
}
}
pub fn is_path(&self) -> bool {
if let Interpolated(ref nt) = *self {
if let NtPath(..) = nt.0 {
return true;
}
}
false
}
pub fn lifetime(&self, span: Span) -> Option<ast::Lifetime> {
match *self {
Lifetime(ident) =>
Some(ast::Lifetime { ident: ident, span: span, id: ast::DUMMY_NODE_ID }),
Interpolated(ref nt) => match nt.0 {
NtLifetime(lifetime) => Some(lifetime),
_ => None,
},
_ => None,
}
}
pub fn is_lifetime(&self) -> bool {
self.lifetime(syntax_pos::DUMMY_SP).is_some()
}
pub fn is_mutability(&self) -> bool {
self.is_keyword(keywords::Mut) ||
self.is_keyword(keywords::Const)
}
pub fn is_qpath_start(&self) -> bool {
self == &Lt || self == &BinOp(Shl)
}
pub fn is_path_start(&self) -> bool {
self == &ModSep || self.is_qpath_start() || self.is_path() ||
self.is_path_segment_keyword() || self.is_ident() && !self.is_reserved_ident()
}
pub fn is_keyword(&self, kw: keywords::Keyword) -> bool {
self.ident().map(|ident| ident.name == kw.name()).unwrap_or(false)
}
pub fn is_path_segment_keyword(&self) -> bool {
match self.ident() {
Some(id) => id.name == keywords::Super.name() ||
id.name == keywords::SelfValue.name() ||
id.name == keywords::SelfType.name() ||
id.name == keywords::Extern.name() ||
id.name == keywords::Crate.name() ||
id.name == keywords::DollarCrate.name(),
None => false,
}
}
pub fn is_special_ident(&self) -> bool {
match self.ident() {
Some(id) => id.name <= keywords::DollarCrate.name(),
_ => false,
}
}
pub fn is_used_keyword(&self) -> bool {
match self.ident() {
Some(id) => id.name >= keywords::As.name() && id.name <= keywords::While.name(),
_ => false,
}
}
pub fn is_unused_keyword(&self) -> bool {
match self.ident() {
Some(id) => id.name >= keywords::Abstract.name() && id.name <= keywords::Yield.name(),
_ => false,
}
}
pub fn glue(self, joint: Token) -> Option<Token> {
Some(match self {
Eq => match joint {
Eq => EqEq,
Gt => FatArrow,
_ => return None,
},
Lt => match joint {
Eq => Le,
Lt => BinOp(Shl),
Le => BinOpEq(Shl),
BinOp(Minus) => LArrow,
_ => return None,
},
Gt => match joint {
Eq => Ge,
Gt => BinOp(Shr),
Ge => BinOpEq(Shr),
_ => return None,
},
Not => match joint {
Eq => Ne,
_ => return None,
},
BinOp(op) => match joint {
Eq => BinOpEq(op),
BinOp(And) if op == And => AndAnd,
BinOp(Or) if op == Or => OrOr,
Gt if op == Minus => RArrow,
_ => return None,
},
Dot => match joint {
Dot => DotDot,
DotDot => DotDotDot,
DotEq => DotDotEq,
_ => return None,
},
DotDot => match joint {
Dot => DotDotDot,
Eq => DotDotEq,
_ => return None,
},
Colon => match joint {
Colon => ModSep,
_ => return None,
},
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | DotEq |
DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar |
Question | OpenDelim(..) | CloseDelim(..) | Underscore => return None,
Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) |
Whitespace | Comment | Shebang(..) | Eof => return None,
})
}
pub fn similar_tokens(&self) -> Option<Vec<Token>> {
match *self {
Comma => Some(vec![Dot, Lt]),
Semi => Some(vec![Colon]),
_ => None
}
}
pub fn is_reserved_ident(&self) -> bool {
self.is_special_ident() || self.is_used_keyword() || self.is_unused_keyword()
}
pub fn interpolated_to_tokenstream(&self, sess: &ParseSess, span: Span)
-> TokenStream
{
let nt = match *self {
Token::Interpolated(ref nt) => nt,
_ => panic!("only works on interpolated tokens"),
};
let mut tokens = None;
match nt.0 {
Nonterminal::NtItem(ref item) => {
tokens = prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span);
}
Nonterminal::NtTraitItem(ref item) => {
tokens = prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span);
}
Nonterminal::NtImplItem(ref item) => {
tokens = prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span);
}
Nonterminal::NtIdent(ident) => {
let token = Token::Ident(ident.node);
tokens = Some(TokenTree::Token(ident.span, token).into());
}
Nonterminal::NtLifetime(lifetime) => {
let token = Token::Lifetime(lifetime.ident);
tokens = Some(TokenTree::Token(lifetime.span, token).into());
}
Nonterminal::NtTT(ref tt) => {
tokens = Some(tt.clone().into());
}
_ => {}
}
tokens.unwrap_or_else(|| {
nt.1.force(|| {
let source = pprust::token_to_string(self);
parse_stream_from_source_str(FileName::MacroExpansion, source, sess, Some(span))
})
})
}
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)]
pub enum Nonterminal {
NtItem(P<ast::Item>),
NtBlock(P<ast::Block>),
NtStmt(ast::Stmt),
NtPat(P<ast::Pat>),
NtExpr(P<ast::Expr>),
NtTy(P<ast::Ty>),
NtIdent(ast::SpannedIdent),
NtMeta(ast::MetaItem),
NtPath(ast::Path),
NtVis(ast::Visibility),
NtTT(TokenTree),
NtArm(ast::Arm),
NtImplItem(ast::ImplItem),
NtTraitItem(ast::TraitItem),
NtGenerics(ast::Generics),
NtWhereClause(ast::WhereClause),
NtArg(ast::Arg),
NtLifetime(ast::Lifetime),
}
impl fmt::Debug for Nonterminal {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
NtItem(..) => f.pad("NtItem(..)"),
NtBlock(..) => f.pad("NtBlock(..)"),
NtStmt(..) => f.pad("NtStmt(..)"),
NtPat(..) => f.pad("NtPat(..)"),
NtExpr(..) => f.pad("NtExpr(..)"),
NtTy(..) => f.pad("NtTy(..)"),
NtIdent(..) => f.pad("NtIdent(..)"),
NtMeta(..) => f.pad("NtMeta(..)"),
NtPath(..) => f.pad("NtPath(..)"),
NtTT(..) => f.pad("NtTT(..)"),
NtArm(..) => f.pad("NtArm(..)"),
NtImplItem(..) => f.pad("NtImplItem(..)"),
NtTraitItem(..) => f.pad("NtTraitItem(..)"),
NtGenerics(..) => f.pad("NtGenerics(..)"),
NtWhereClause(..) => f.pad("NtWhereClause(..)"),
NtArg(..) => f.pad("NtArg(..)"),
NtVis(..) => f.pad("NtVis(..)"),
NtLifetime(..) => f.pad("NtLifetime(..)"),
}
}
}
pub fn is_op(tok: &Token) -> bool {
match *tok {
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) |
Ident(..) | Underscore | Lifetime(..) | Interpolated(..) |
Whitespace | Comment | Shebang(..) | Eof => false,
_ => true,
}
}
pub struct LazyTokenStream(Cell<Option<TokenStream>>);
impl Clone for LazyTokenStream {
fn clone(&self) -> Self {
let opt_stream = self.0.take();
self.0.set(opt_stream.clone());
LazyTokenStream(Cell::new(opt_stream))
}
}
impl cmp::Eq for LazyTokenStream {}
impl PartialEq for LazyTokenStream {
fn eq(&self, _other: &LazyTokenStream) -> bool {
true
}
}
impl fmt::Debug for LazyTokenStream {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
fmt::Debug::fmt(&self.clone().0.into_inner(), f)
}
}
impl LazyTokenStream {
pub fn new() -> Self {
LazyTokenStream(Cell::new(None))
}
pub fn force<F: FnOnce() -> TokenStream>(&self, f: F) -> TokenStream {
let mut opt_stream = self.0.take();
if opt_stream.is_none() {
opt_stream = Some(f());
}
self.0.set(opt_stream.clone());
opt_stream.clone().unwrap()
}
}
impl Encodable for LazyTokenStream {
fn encode<S: Encoder>(&self, _: &mut S) -> Result<(), S::Error> {
Ok(())
}
}
impl Decodable for LazyTokenStream {
fn decode<D: Decoder>(_: &mut D) -> Result<LazyTokenStream, D::Error> {
Ok(LazyTokenStream::new())
}
}
impl ::std::hash::Hash for LazyTokenStream {
fn hash<H: ::std::hash::Hasher>(&self, _hasher: &mut H) {}
}
fn prepend_attrs(sess: &ParseSess,
attrs: &[ast::Attribute],
tokens: Option<&tokenstream::TokenStream>,
span: syntax_pos::Span)
-> Option<tokenstream::TokenStream>
{
let tokens = tokens?;
if attrs.len() == 0 {
return Some(tokens.clone())
}
let mut builder = tokenstream::TokenStreamBuilder::new();
for attr in attrs {
assert_eq!(attr.style, ast::AttrStyle::Outer,
"inner attributes should prevent cached tokens from existing");
let name = FileName::MacroExpansion;
let source = pprust::attr_to_string(attr);
let stream = parse_stream_from_source_str(name, source, sess, Some(span));
builder.push(stream);
}
builder.push(tokens.clone());
Some(builder.build())
}