use ast;
use ext::tt::macro_parser;
use parse::{ParseSess, token};
use print::pprust;
use symbol::{keywords, Symbol};
use syntax_pos::{DUMMY_SP, Span, BytePos};
use tokenstream;
use std::rc::Rc;
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Debug)]
pub struct Delimited {
pub delim: token::DelimToken,
pub tts: Vec<TokenTree>,
}
impl Delimited {
pub fn open_token(&self) -> token::Token {
token::OpenDelim(self.delim)
}
pub fn close_token(&self) -> token::Token {
token::CloseDelim(self.delim)
}
pub fn open_tt(&self, span: Span) -> TokenTree {
let open_span = if span == DUMMY_SP {
DUMMY_SP
} else {
Span { hi: span.lo + BytePos(self.delim.len() as u32), ..span }
};
TokenTree::Token(open_span, self.open_token())
}
pub fn close_tt(&self, span: Span) -> TokenTree {
let close_span = if span == DUMMY_SP {
DUMMY_SP
} else {
Span { lo: span.hi - BytePos(self.delim.len() as u32), ..span }
};
TokenTree::Token(close_span, self.close_token())
}
}
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Debug)]
pub struct SequenceRepetition {
pub tts: Vec<TokenTree>,
pub separator: Option<token::Token>,
pub op: KleeneOp,
pub num_captures: usize,
}
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Debug, Copy)]
pub enum KleeneOp {
ZeroOrMore,
OneOrMore,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)]
pub enum TokenTree {
Token(Span, token::Token),
Delimited(Span, Rc<Delimited>),
Sequence(Span, Rc<SequenceRepetition>),
MetaVarDecl(Span, ast::Ident , ast::Ident ),
}
impl TokenTree {
pub fn len(&self) -> usize {
match *self {
TokenTree::Delimited(_, ref delimed) => match delimed.delim {
token::NoDelim => delimed.tts.len(),
_ => delimed.tts.len() + 2,
},
TokenTree::Sequence(_, ref seq) => seq.tts.len(),
_ => 0,
}
}
pub fn is_empty(&self) -> bool {
match *self {
TokenTree::Delimited(_, ref delimed) => match delimed.delim {
token::NoDelim => delimed.tts.is_empty(),
_ => false,
},
TokenTree::Sequence(_, ref seq) => seq.tts.is_empty(),
_ => true,
}
}
pub fn get_tt(&self, index: usize) -> TokenTree {
match (self, index) {
(&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => {
delimed.tts[index].clone()
}
(&TokenTree::Delimited(span, ref delimed), _) => {
if index == 0 {
return delimed.open_tt(span);
}
if index == delimed.tts.len() + 1 {
return delimed.close_tt(span);
}
delimed.tts[index - 1].clone()
}
(&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(),
_ => panic!("Cannot expand a token tree"),
}
}
pub fn span(&self) -> Span {
match *self {
TokenTree::Token(sp, _) |
TokenTree::MetaVarDecl(sp, _, _) |
TokenTree::Delimited(sp, _) |
TokenTree::Sequence(sp, _) => sp,
}
}
}
pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &ParseSess)
-> Vec<TokenTree> {
let mut result = Vec::new();
let mut trees = input.trees();
while let Some(tree) = trees.next() {
let tree = parse_tree(tree, &mut trees, expect_matchers, sess);
match tree {
TokenTree::Token(start_sp, token::SubstNt(ident)) if expect_matchers => {
let span = match trees.next() {
Some(tokenstream::TokenTree::Token(span, token::Colon)) => match trees.next() {
Some(tokenstream::TokenTree::Token(end_sp, ref tok)) => match tok.ident() {
Some(kind) => {
let span = Span { lo: start_sp.lo, ..end_sp };
result.push(TokenTree::MetaVarDecl(span, ident, kind));
continue
}
_ => end_sp,
},
tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
},
tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp),
};
sess.missing_fragment_specifiers.borrow_mut().insert(span);
result.push(TokenTree::MetaVarDecl(span, ident, keywords::Invalid.ident()));
}
_ => result.push(tree),
}
}
result
}
fn parse_tree<I>(tree: tokenstream::TokenTree,
trees: &mut I,
expect_matchers: bool,
sess: &ParseSess)
-> TokenTree
where I: Iterator<Item = tokenstream::TokenTree>,
{
match tree {
tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
Some(tokenstream::TokenTree::Delimited(span, delimited)) => {
if delimited.delim != token::Paren {
let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
let msg = format!("expected `(`, found `{}`", tok);
sess.span_diagnostic.span_err(span, &msg);
}
let sequence = parse(delimited.tts.into(), expect_matchers, sess);
let (separator, op) = parse_sep_and_kleene_op(trees, span, sess);
let name_captures = macro_parser::count_names(&sequence);
TokenTree::Sequence(span, Rc::new(SequenceRepetition {
tts: sequence,
separator: separator,
op: op,
num_captures: name_captures,
}))
}
Some(tokenstream::TokenTree::Token(ident_span, token::Ident(ident))) => {
let span = Span { lo: span.lo, ..ident_span };
if ident.name == keywords::Crate.name() {
let ident = ast::Ident { name: Symbol::intern("$crate"), ..ident };
TokenTree::Token(span, token::Ident(ident))
} else {
TokenTree::Token(span, token::SubstNt(ident))
}
}
Some(tokenstream::TokenTree::Token(span, tok)) => {
let msg = format!("expected identifier, found `{}`", pprust::token_to_string(&tok));
sess.span_diagnostic.span_err(span, &msg);
TokenTree::Token(span, token::SubstNt(keywords::Invalid.ident()))
}
None => TokenTree::Token(span, token::Dollar),
},
tokenstream::TokenTree::Token(span, tok) => TokenTree::Token(span, tok),
tokenstream::TokenTree::Delimited(span, delimited) => {
TokenTree::Delimited(span, Rc::new(Delimited {
delim: delimited.delim,
tts: parse(delimited.tts.into(), expect_matchers, sess),
}))
}
}
}
fn parse_sep_and_kleene_op<I>(input: &mut I, span: Span, sess: &ParseSess)
-> (Option<token::Token>, KleeneOp)
where I: Iterator<Item = tokenstream::TokenTree>,
{
fn kleene_op(token: &token::Token) -> Option<KleeneOp> {
match *token {
token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
_ => None,
}
}
let span = match input.next() {
Some(tokenstream::TokenTree::Token(span, tok)) => match kleene_op(&tok) {
Some(op) => return (None, op),
None => match input.next() {
Some(tokenstream::TokenTree::Token(span, tok2)) => match kleene_op(&tok2) {
Some(op) => return (Some(tok), op),
None => span,
},
tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
}
},
tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
};
sess.span_diagnostic.span_err(span, "expected `*` or `+`");
(None, KleeneOp::ZeroOrMore)
}