use std::borrow::Cow;
use std::cell::OnceCell;
use std::ops::Range;
use unicode_normalization::UnicodeNormalization;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Edition {
Rust2015,
Rust2018,
Rust2021,
}
impl Edition {
pub(crate) fn of(manifest_edition: Option<&str>) -> Self {
match manifest_edition {
Some("2015") => Edition::Rust2015,
Some("2018") => Edition::Rust2018,
_ => Edition::Rust2021,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum Delimiter {
Parenthesis,
Bracket,
Brace,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum Kind {
Ident,
RawIdent,
Keyword,
Lifetime,
Literal,
Punct,
Open(Delimiter),
Close(Delimiter),
End,
}
#[derive(Clone, Debug)]
pub(super) struct Token {
pub kind: Kind,
pub span: Range<usize>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum Node {
Token(usize),
Group {
open: usize,
close: usize,
},
Macro {
name: usize,
bang: usize,
open: usize,
close: usize,
},
Attribute {
hash: usize,
open: usize,
close: usize,
},
}
impl Node {
pub(super) fn first(self) -> usize {
match self {
Node::Token(i) => i,
Node::Group { open, .. } => open,
Node::Macro { name, .. } => name,
Node::Attribute { hash, .. } => hash,
}
}
pub(super) fn last(self) -> usize {
match self {
Node::Token(i) => i,
Node::Group { close, .. }
| Node::Macro { close, .. }
| Node::Attribute { close, .. } => close,
}
}
}
pub(super) struct TokenTree<'s> {
source: Cow<'s, str>,
tokens: Vec<Token>,
partner: Vec<usize>,
parent: Vec<Option<usize>>,
angles: OnceCell<Vec<AngleSlot>>,
}
#[derive(Clone, Copy, Default)]
pub(super) struct AngleSlot {
pub(super) close: Option<usize>,
pub(super) inner_close: Option<usize>,
pub(super) enclosed: bool,
}
const RESERVED_WORDS: [&str; 52] = [
"as", "break", "const", "continue", "crate", "else", "enum", "extern", "false", "fn", "for",
"if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", "ref", "return",
"self", "Self", "static", "struct", "super", "trait", "true", "type", "unsafe", "use", "where",
"while", "async", "await", "dyn", "abstract", "become", "box", "do", "final", "macro",
"override", "priv", "typeof", "unsized", "virtual", "yield", "try", "gen",
];
fn is_reserved_word(word: &str) -> bool {
RESERVED_WORDS.contains(&word)
}
pub(super) fn is_keyword(word: &str, edition: Edition) -> bool {
is_reserved_word(word)
&& word != "gen"
&& !(edition == Edition::Rust2015 && matches!(word, "async" | "await" | "dyn" | "try"))
}
pub(super) const MULTI_BYTE_PUNCTUATION: [&str; 25] = [
"...", "..=", "<<=", ">>=", "!=", "%=", "&&", "&=", "*=", "+=", "-=", "->", "..", "/=", "::",
"<-", "<<", "<=", "==", "=>", ">=", ">>", "^=", "|=", "||",
];
fn source_start(bytes: &[u8]) -> usize {
let bom = if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
3
} else {
0
};
bom + shebang_end(&bytes[bom..])
}
fn shebang_end(bytes: &[u8]) -> usize {
if !bytes.starts_with(b"#!") {
return 0;
}
let mut k = 2;
loop {
match bytes.get(k) {
Some(_) if white_space_len(bytes, k) > 0 => k += white_space_len(bytes, k),
Some(b'/') if bytes.get(k + 1) == Some(&b'/') => {
while k < bytes.len() && bytes[k] != b'\n' {
k += 1;
}
}
Some(b'/') if bytes.get(k + 1) == Some(&b'*') => k = block_comment_end(bytes, k),
Some(b'[') => return 0,
_ => break,
}
}
bytes
.iter()
.position(|&b| b == b'\n')
.map_or(bytes.len(), |at| at + 1)
}
pub(super) fn white_space_len(bytes: &[u8], i: usize) -> usize {
const WIDE: [&[u8]; 5] = [
"\u{85}".as_bytes(),
"\u{200e}".as_bytes(),
"\u{200f}".as_bytes(),
"\u{2028}".as_bytes(),
"\u{2029}".as_bytes(),
];
match bytes.get(i) {
Some(b'\t' | b'\n' | b'\x0b' | b'\x0c' | b'\r' | b' ') => 1,
Some(_) => WIDE
.iter()
.find(|wide| bytes[i..].starts_with(wide))
.map_or(0, |wide| wide.len()),
None => 0,
}
}
pub(super) fn is_ident_byte(byte: u8) -> bool {
byte == b'_' || byte.is_ascii_alphanumeric() || byte >= 0x80
}
fn is_ident_start(byte: u8) -> bool {
byte == b'_' || byte.is_ascii_alphabetic() || byte >= 0x80
}
impl<'s> TokenTree<'s> {
pub(super) fn lex(source: &'s str, edition: Edition) -> Self {
let bytes = source.as_bytes();
let mut tokens: Vec<Token> = Vec::new();
let mut i = source_start(bytes);
while i < bytes.len() {
let b = bytes[i];
let space = white_space_len(bytes, i);
if space > 0 {
i += space;
continue;
}
if b == b'/' && bytes.get(i + 1) == Some(&b'/') {
while i < bytes.len() && bytes[i] != b'\n' {
i += 1;
}
continue;
}
if b == b'/' && bytes.get(i + 1) == Some(&b'*') {
i = block_comment_end(bytes, i);
continue;
}
let (kind, end) = if let Some(end) = literal_end(bytes, i, edition) {
(Kind::Literal, end)
} else if b == b'r'
&& bytes.get(i + 1) == Some(&b'#')
&& bytes.get(i + 2).is_some_and(|&c| is_ident_start(c))
{
(Kind::RawIdent, word_end(bytes, i + 2))
} else if b == b'\'' {
match lifetime_end(bytes, i) {
Some(end) => (Kind::Lifetime, end),
None => (Kind::Punct, i + 1),
}
} else if b.is_ascii_digit() {
(Kind::Literal, number_end(bytes, i))
} else if is_ident_start(b) {
let end = word_end(bytes, i);
let word = &source[i..end];
let kind = if word == "_" {
Kind::Punct
} else if is_keyword(word, edition) {
Kind::Keyword
} else {
Kind::Ident
};
(kind, end)
} else {
let kind = match b {
b'(' => Kind::Open(Delimiter::Parenthesis),
b'[' => Kind::Open(Delimiter::Bracket),
b'{' => Kind::Open(Delimiter::Brace),
b')' => Kind::Close(Delimiter::Parenthesis),
b']' => Kind::Close(Delimiter::Bracket),
b'}' => Kind::Close(Delimiter::Brace),
_ => Kind::Punct,
};
let len = if kind == Kind::Punct {
MULTI_BYTE_PUNCTUATION
.iter()
.find(|p| bytes[i..].starts_with(p.as_bytes()))
.map_or(1, |p| p.len())
} else {
1
};
(kind, i + len)
};
tokens.push(Token {
kind,
span: i..end.min(bytes.len()),
});
i = end.max(i + 1);
}
let mut partner: Vec<usize> = (0..tokens.len()).collect();
let mut stack: Vec<(usize, Delimiter)> = Vec::new();
for k in 0..tokens.len() {
match tokens[k].kind {
Kind::Open(d) => stack.push((k, d)),
Kind::Close(d) => match stack.last() {
Some(&(open, top)) if top == d => {
stack.pop();
partner[open] = k;
partner[k] = open;
}
_ => tokens[k].kind = Kind::Punct,
},
_ => {}
}
}
while let Some((open, d)) = stack.pop() {
let k = tokens.len();
tokens.push(Token {
kind: Kind::Close(d),
span: bytes.len()..bytes.len(),
});
partner.push(open);
partner[open] = k;
}
let mut parent = vec![None; tokens.len()];
let mut open_stack: Vec<usize> = Vec::new();
for k in 0..tokens.len() {
if let Kind::Close(_) = tokens[k].kind {
open_stack.pop();
}
parent[k] = open_stack.last().copied();
if let Kind::Open(_) = tokens[k].kind {
open_stack.push(k);
}
}
TokenTree {
source: identifiers_in_nfc(source, &mut tokens),
tokens,
partner,
parent,
angles: OnceCell::new(),
}
}
pub(super) fn angle_slots(&self, read: impl FnOnce() -> Vec<AngleSlot>) -> &[AngleSlot] {
self.angles.get_or_init(read)
}
pub(super) fn len(&self) -> usize {
self.tokens.len()
}
pub(super) fn kind(&self, i: usize) -> Kind {
self.tokens.get(i).map_or(Kind::End, |token| token.kind)
}
pub(super) fn text(&self, i: usize) -> &str {
let span = self.tokens[i].span.clone();
let text = &self.source[span];
if self.tokens[i].kind == Kind::RawIdent {
&text[2..]
} else {
text
}
}
pub(super) fn written(&self, i: usize) -> &str {
&self.source[self.tokens[i].span.clone()]
}
pub(super) fn is(&self, i: usize, text: &str) -> bool {
i < self.tokens.len()
&& matches!(
self.tokens[i].kind,
Kind::Punct | Kind::Keyword | Kind::Ident
)
&& self.text(i) == text
}
pub(super) fn opens_a_macro_rules(&self, m: usize) -> bool {
self.is(m, "macro_rules") && self.is(m + 1, "!") && self.is_word(m + 2)
}
pub(super) fn is_word(&self, i: usize) -> bool {
i < self.tokens.len() && matches!(self.tokens[i].kind, Kind::Ident | Kind::RawIdent)
}
#[cfg(test)]
pub(super) fn start(&self, i: usize) -> usize {
self.tokens[i].span.start
}
pub(super) fn string_value(&self, i: usize) -> Option<String> {
if self.tokens.get(i)?.kind != Kind::Literal {
return None;
}
let text = self.written(i);
if let Some(raw) = text.strip_prefix('r') {
let hashes = raw.len() - raw.trim_start_matches('#').len();
let inner = raw[hashes..].strip_prefix('"')?;
return inner
.strip_suffix(&format!("\"{}", "#".repeat(hashes)))
.map(str::to_string);
}
let inner = text.strip_prefix('"')?.strip_suffix('"')?;
decode_str_escapes(inner.as_bytes())
}
pub(super) fn partner(&self, i: usize) -> usize {
self.partner[i]
}
pub(super) fn enclosing(&self, i: usize) -> Option<usize> {
self.parent[i]
}
pub(super) fn node_at(&self, i: usize) -> Node {
match self.kind(i) {
Kind::Open(_) => Node::Group {
open: i,
close: self.partner[i],
},
Kind::Punct if self.text(i) == "#" => {
let open = if self.is(i + 1, "!") { i + 2 } else { i + 1 };
if self.kind(open) == Kind::Open(Delimiter::Bracket) {
Node::Attribute {
hash: i,
open,
close: self.partner[open],
}
} else {
Node::Token(i)
}
}
Kind::Ident | Kind::RawIdent => {
let open = if self.opens_a_macro_rules(i) {
i + 3
} else {
i + 2
};
if self.is(i + 1, "!")
&& open < self.len()
&& matches!(self.kind(open), Kind::Open(_))
{
Node::Macro {
name: i,
bang: i + 1,
open,
close: self.partner[open],
}
} else {
Node::Token(i)
}
}
_ => Node::Token(i),
}
}
pub(super) fn node_before(&self, i: usize) -> Option<Node> {
if i == 0 {
return None;
}
let j = i - 1;
if let Kind::Open(_) = self.kind(j) {
return None;
}
let Kind::Close(_) = self.kind(j) else {
return Some(Node::Token(j));
};
let open = self.partner[j];
let before = |k: usize| open.checked_sub(k);
if self.kind(open) == Kind::Open(Delimiter::Bracket) {
if let Some(h) = before(1).filter(|&h| self.is(h, "#")) {
return Some(Node::Attribute {
hash: h,
open,
close: j,
});
}
if let Some(h) = before(2).filter(|&h| self.is(h + 1, "!") && self.is(h, "#")) {
return Some(Node::Attribute {
hash: h,
open,
close: j,
});
}
}
if let Some(bang) = before(1).filter(|&b| self.is(b, "!")) {
if let Some(name) = bang.checked_sub(1).filter(|&n| self.is_word(n)) {
return Some(Node::Macro {
name,
bang,
open,
close: j,
});
}
}
if let Some(m) = before(3).filter(|&m| self.opens_a_macro_rules(m)) {
return Some(Node::Macro {
name: m,
bang: m + 1,
open,
close: j,
});
}
Some(Node::Group { open, close: j })
}
}
fn identifiers_in_nfc<'s>(source: &'s str, tokens: &mut [Token]) -> Cow<'s, str> {
let is_identifier = |token: &Token| matches!(token.kind, Kind::Ident | Kind::RawIdent);
let unnormalized = |token: &Token| {
is_identifier(token) && {
let text = &source[token.span.clone()];
!text.is_ascii() && !unicode_normalization::is_nfc(text)
}
};
if !tokens.iter().any(unnormalized) {
return Cow::Borrowed(source);
}
let mut normalized = String::with_capacity(source.len());
let mut copied = 0;
for token in tokens {
let span = token.span.clone();
normalized.push_str(&source[copied..span.start]);
let start = normalized.len();
if is_identifier(token) {
normalized.extend(source[span.clone()].nfc());
} else {
normalized.push_str(&source[span.clone()]);
}
token.span = start..normalized.len();
copied = span.end;
}
normalized.push_str(&source[copied..]);
Cow::Owned(normalized)
}
fn decode_str_escapes(inner: &[u8]) -> Option<String> {
let s = std::str::from_utf8(inner).ok()?;
let mut out = String::with_capacity(s.len());
let mut chars = s.chars().peekable();
while let Some(c) = chars.next() {
if c != '\\' {
out.push(c);
continue;
}
match chars.next()? {
'n' => out.push('\n'),
'r' => out.push('\r'),
't' => out.push('\t'),
'\\' => out.push('\\'),
'0' => out.push('\0'),
'\'' => out.push('\''),
'"' => out.push('"'),
'\r' | '\n' => {
while matches!(chars.peek(), Some(' ' | '\t' | '\n' | '\r')) {
chars.next();
}
}
'x' => {
let hi = chars.next()?.to_digit(16)?;
let lo = chars.next()?.to_digit(16)?;
let v = hi * 16 + lo;
if v > 0x7F {
return None;
}
out.push(char::from_u32(v)?);
}
'u' => {
if chars.next()? != '{' {
return None;
}
let mut value: u32 = 0;
let mut digits = 0;
loop {
match chars.next()? {
'}' => break,
'_' if digits == 0 => return None,
'_' => continue,
d => {
let hd = d.to_digit(16)?;
digits += 1;
if digits > 6 {
return None;
}
value = value * 16 + hd;
}
}
}
if digits == 0 {
return None;
}
out.push(char::from_u32(value)?);
}
_ => return None,
}
}
Some(out)
}
fn utf8_len(lead: u8) -> usize {
match lead {
0xC0..=0xDF => 2,
0xE0..=0xEF => 3,
0xF0..=0xF7 => 4,
_ => 1,
}
}
fn word_end(bytes: &[u8], mut i: usize) -> usize {
while i < bytes.len() && is_ident_byte(bytes[i]) && white_space_len(bytes, i) == 0 {
i += 1;
}
i
}
fn block_comment_end(bytes: &[u8], mut i: usize) -> usize {
i += 2;
let mut depth = 1usize;
while i < bytes.len() && depth > 0 {
if bytes[i] == b'/' && bytes.get(i + 1) == Some(&b'*') {
depth += 1;
i += 2;
} else if bytes[i] == b'*' && bytes.get(i + 1) == Some(&b'/') {
depth -= 1;
i += 2;
} else {
i += 1;
}
}
i.min(bytes.len())
}
fn literal_end(bytes: &[u8], i: usize, edition: Edition) -> Option<usize> {
let mut j = i;
if bytes.get(j) == Some(&b'b') || (bytes.get(j) == Some(&b'c') && edition == Edition::Rust2021)
{
j += 1;
}
if bytes.get(j) == Some(&b'r') {
let mut k = j + 1;
let mut hashes = 0;
while bytes.get(k) == Some(&b'#') {
hashes += 1;
k += 1;
}
if bytes.get(k) == Some(&b'"') {
let mut m = k + 1;
while m < bytes.len() {
if bytes[m] == b'"' && (0..hashes).all(|h| bytes.get(m + 1 + h) == Some(&b'#')) {
return Some(m + 1 + hashes);
}
m += 1;
}
return Some(bytes.len());
}
}
match bytes.get(j) {
Some(b'"') => {
let mut m = j + 1;
while m < bytes.len() && bytes[m] != b'"' {
m += if bytes[m] == b'\\' { 2 } else { 1 };
}
Some((m + 1).min(bytes.len()))
}
Some(b'\'') if j == i || bytes[i] == b'b' => char_literal_end(bytes, j),
_ => None,
}
}
fn char_literal_end(bytes: &[u8], q: usize) -> Option<usize> {
if bytes.get(q + 1) == Some(&b'\\') {
let mut m = q + 3;
while m < bytes.len() && bytes[m] != b'\'' {
m += 1;
}
return Some((m + 1).min(bytes.len()));
}
let lead = *bytes.get(q + 1)?;
let len = utf8_len(lead);
(bytes.get(q + 1 + len) == Some(&b'\'')).then_some(q + 2 + len)
}
fn lifetime_end(bytes: &[u8], q: usize) -> Option<usize> {
let start = if bytes.get(q + 1) == Some(&b'r') && bytes.get(q + 2) == Some(&b'#') {
q + 3
} else {
q + 1
};
bytes
.get(start)
.is_some_and(|&b| is_ident_start(b))
.then(|| word_end(bytes, start))
}
fn number_end(bytes: &[u8], i: usize) -> usize {
let prefixed = bytes[i] == b'0' && matches!(bytes.get(i + 1), Some(b'x' | b'o' | b'b'));
let mut j = word_end(bytes, i);
let exponent_sign = |j: usize| {
!prefixed
&& matches!(bytes[j - 1], b'e' | b'E')
&& matches!(bytes.get(j), Some(b'+' | b'-'))
&& bytes.get(j + 1).is_some_and(u8::is_ascii_digit)
};
if exponent_sign(j) {
j = word_end(bytes, j + 1);
}
let digits_only = bytes[i..j].iter().all(|&b| b.is_ascii_digit() || b == b'_');
if !prefixed
&& digits_only
&& bytes.get(j) == Some(&b'.')
&& bytes
.get(j + 1)
.is_none_or(|&next| next != b'.' && !is_ident_start(next))
{
j = if bytes.get(j + 1).is_some_and(u8::is_ascii_digit) {
word_end(bytes, j + 1)
} else {
j + 1
};
if exponent_sign(j) {
j = word_end(bytes, j + 1);
}
}
j
}
#[cfg(test)]
mod tests {
use super::*;
fn kinds(source: &str) -> Vec<(Kind, String)> {
let tree = TokenTree::lex(source, Edition::Rust2021);
(0..tree.len())
.map(|i| (tree.kind(i), tree.written(i).to_string()))
.collect()
}
#[test]
fn an_identifier_is_read_in_nfc_and_a_literal_as_written() {
let decomposed = "crate::se\u{301}cret::go(\"se\u{301}\"); r#se\u{301}cret";
assert_eq!(
kinds(decomposed),
vec![
(Kind::Keyword, "crate".into()),
(Kind::Punct, "::".into()),
(Kind::Ident, "s\u{e9}cret".into()),
(Kind::Punct, "::".into()),
(Kind::Ident, "go".into()),
(Kind::Open(Delimiter::Parenthesis), "(".into()),
(Kind::Literal, "\"se\u{301}\"".into()),
(Kind::Close(Delimiter::Parenthesis), ")".into()),
(Kind::Punct, ";".into()),
(Kind::RawIdent, "r#s\u{e9}cret".into()),
]
);
}
#[test]
fn a_shebang_line_is_no_token_and_an_inner_attribute_is_one() {
let tree = TokenTree::lex("#!/usr/bin/env run\nmod core;", Edition::Rust2018);
assert_eq!(tree.text(0), "mod", "the shebang line is skipped");
let tree = TokenTree::lex("\u{feff}mod core;", Edition::Rust2018);
assert_eq!(
tree.text(0),
"mod",
"a byte-order mark is no part of the first token"
);
for source in [
"#![allow(dead_code)]\nmod core;",
"#! /* note */ [allow(dead_code)]\nmod core;",
"#! /* /* nested */ */ [allow(dead_code)] mod core;",
] {
let tree = TokenTree::lex(source, Edition::Rust2018);
assert_eq!(
tree.text(0),
"#",
"{source:?} opens with an inner attribute, no shebang"
);
}
}
#[test]
fn the_punctuation_set_is_the_references_both_ways() {
let reference = [
"+=", "-=", "*=", "/=", "%=", "^=", "&=", "|=", "<<=", ">>=", "==", "!=", ">=", "<=",
"&&", "||", "<<", ">>", "..", "...", "..=", "::", "->", "=>", "<-",
];
for token in reference {
assert!(MULTI_BYTE_PUNCTUATION.contains(&token), "{token} missing");
}
for token in MULTI_BYTE_PUNCTUATION {
assert!(
reference.contains(&token),
"{token} is not in the Reference's table"
);
}
for (i, a) in MULTI_BYTE_PUNCTUATION.iter().enumerate() {
for b in &MULTI_BYTE_PUNCTUATION[i + 1..] {
assert!(
!(b.starts_with(a) && b.len() > a.len()),
"{b} is shadowed by {a}"
);
}
}
}
#[test]
fn maximal_munch_reads_each_operator_whole() {
assert_eq!(
kinds("a<<=b..=c->d"),
[
(Kind::Ident, "a".into()),
(Kind::Punct, "<<=".into()),
(Kind::Ident, "b".into()),
(Kind::Punct, "..=".into()),
(Kind::Ident, "c".into()),
(Kind::Punct, "->".into()),
(Kind::Ident, "d".into()),
]
);
}
#[test]
fn a_char_literal_is_one_literal_token() {
for source in [
"'a'",
"'\\n'",
"'\\u{1F600}'",
"'未'",
"b'x'",
"'\"'",
"'}'",
] {
assert_eq!(
kinds(source),
[(Kind::Literal, source.to_string())],
"{source}"
);
}
assert_eq!(
kinds("'a 'static"),
[
(Kind::Lifetime, "'a".into()),
(Kind::Lifetime, "'static".into())
]
);
}
#[test]
fn every_string_form_is_one_literal_token() {
for source in [
"\"a { // \\\" b\"",
"r\"x\"",
"r#\"a \" } \"#",
"br##\"x\"##",
"cr#\"}\"#",
"b\"x\"",
"c\"x\"",
] {
assert_eq!(
kinds(source),
[(Kind::Literal, source.to_string())],
"{source}"
);
}
}
#[test]
fn a_c_before_a_string_is_an_identifier_before_2021() {
let tree = TokenTree::lex("c\"x\"", Edition::Rust2018);
assert_eq!(
(0..tree.len()).map(|i| tree.kind(i)).collect::<Vec<_>>(),
[Kind::Ident, Kind::Literal]
);
}
#[test]
fn a_number_is_one_literal_token() {
assert_eq!(kinds("1u8"), [(Kind::Literal, "1u8".into())]);
assert_eq!(kinds("1.5e-3f64"), [(Kind::Literal, "1.5e-3f64".into())]);
assert_eq!(kinds("0x1e"), [(Kind::Literal, "0x1e".into())]);
assert_eq!(
kinds("1..2"),
[
(Kind::Literal, "1".into()),
(Kind::Punct, "..".into()),
(Kind::Literal, "2".into())
]
);
assert_eq!(
kinds("1.max"),
[
(Kind::Literal, "1".into()),
(Kind::Punct, ".".into()),
(Kind::Ident, "max".into())
]
);
}
#[test]
fn comments_nest_and_leave_no_token() {
assert_eq!(
kinds("a /* b /* c */ d */ e // f\ng"),
[
(Kind::Ident, "a".into()),
(Kind::Ident, "e".into()),
(Kind::Ident, "g".into()),
]
);
}
#[test]
fn a_keyword_is_read_by_edition() {
let tree = TokenTree::lex("async dyn r#match gen _", Edition::Rust2015);
assert_eq!(
(0..tree.len()).map(|i| tree.kind(i)).collect::<Vec<_>>(),
[
Kind::Ident,
Kind::Ident,
Kind::RawIdent,
Kind::Ident,
Kind::Punct
]
);
assert_eq!(tree.text(2), "match");
let tree = TokenTree::lex("async dyn", Edition::Rust2018);
assert_eq!(tree.kind(0), Kind::Keyword);
assert_eq!(tree.kind(1), Kind::Keyword);
}
#[test]
fn partners_pair_only_matching_delimiters() {
let tree = TokenTree::lex("( [ ) ] }", Edition::Rust2018);
assert_eq!(tree.partner(1), 3, "`[` closes at `]`");
assert_eq!(
tree.kind(2),
Kind::Punct,
"a `)` inside `[…]` closes nothing"
);
assert_eq!(
tree.kind(4),
Kind::Punct,
"a stray closing brace closes nothing"
);
assert_eq!(
tree.kind(tree.partner(0)),
Kind::Close(Delimiter::Parenthesis)
);
assert_eq!(
tree.partner(0),
tree.len() - 1,
"an unclosed `(` closes at the end of the text"
);
}
#[test]
fn a_mismatched_closer_closes_nothing() {
let tree = TokenTree::lex("{ ) }", Edition::Rust2018);
assert_eq!(tree.partner(0), 2);
assert_eq!(tree.kind(1), Kind::Punct);
}
#[test]
fn a_macro_call_and_an_attribute_are_one_node() {
let tree = TokenTree::lex(
"thread_local! { x } #![a] #[b] mod m {} return !(x) macro_rules! m { }",
Edition::Rust2018,
);
assert!(matches!(
tree.node_at(0),
Node::Macro {
name: 0,
bang: 1,
open: 2,
close: 4
}
));
assert!(matches!(
tree.node_at(5),
Node::Attribute {
hash: 5,
open: 7,
..
}
));
assert!(matches!(
tree.node_before(14),
Some(Node::Attribute { hash: 10, .. })
));
assert!(matches!(
tree.node_before(10),
Some(Node::Attribute { hash: 5, .. })
));
assert!(matches!(
tree.node_before(5),
Some(Node::Macro { name: 0, .. })
));
let ret = (0..tree.len()).find(|&i| tree.is(i, "return")).unwrap();
assert!(
matches!(tree.node_at(ret + 1), Node::Token(_)),
"a keyword before `!` is a negation"
);
let mr = (0..tree.len())
.find(|&i| tree.is(i, "macro_rules"))
.unwrap();
assert!(matches!(tree.node_at(mr), Node::Macro { .. }));
assert!(
matches!(tree.node_before(tree.len()), Some(Node::Macro { name, .. }) if name == mr)
);
}
#[test]
fn odd_input_is_read_without_panicking() {
for source in [
"r#\"", "\"", "/*", "'", "r", "b'", "c\"", "0x", "1e+", "'r#", "#", "#!", "a!", "}",
"(", "",
] {
let tree = TokenTree::lex(source, Edition::Rust2018);
for i in 0..tree.len() {
let _ = (tree.node_at(i), tree.node_before(i), tree.text(i));
}
}
}
}