#![doc = include_str!("../README.md")]
#![allow(
clippy::derive_partial_eq_without_eq,
clippy::doc_markdown,
clippy::match_same_arms,
clippy::module_name_repetitions,
clippy::needless_doctest_main,
clippy::too_many_lines
)]
#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
extern crate proc_macro;
mod attr;
mod error;
mod segment;
#[cfg(coverage_nightly)]
mod test_helpers;
use crate::attr::expand_attr;
use crate::error::{Error, Result};
use crate::segment::Segment;
use proc_macro::{
Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree,
};
use std::char;
use std::iter;
use std::panic;
use std::str::FromStr;
#[proc_macro]
pub fn paste(input: TokenStream) -> TokenStream {
let mut contains_paste = false;
let flatten_single_interpolation = true;
match expand(
input.clone(),
&mut contains_paste,
flatten_single_interpolation,
) {
Ok(expanded) => {
if contains_paste {
expanded
} else {
input
}
}
Err(err) => err.to_compile_error(),
}
}
#[doc(hidden)]
#[proc_macro]
pub fn item(input: TokenStream) -> TokenStream {
paste(input)
}
#[doc(hidden)]
#[proc_macro]
pub fn expr(input: TokenStream) -> TokenStream {
paste(input)
}
fn expand(
input: TokenStream,
contains_paste: &mut bool,
flatten_single_interpolation: bool,
) -> Result<TokenStream> {
let mut expanded = TokenStream::new();
let mut lookbehind = Lookbehind::Other;
let mut prev_none_group = None::<Group>;
let mut tokens = input.into_iter().peekable();
loop {
let token = tokens.next();
if let Some(group) = prev_none_group.take() {
if match (&token, tokens.peek()) {
(Some(TokenTree::Punct(fst)), Some(TokenTree::Punct(snd))) => {
fst.as_char() == ':' && snd.as_char() == ':' && fst.spacing() == Spacing::Joint
}
_ => false,
} {
expanded.extend(group.stream());
*contains_paste = true;
} else {
expanded.extend(iter::once(TokenTree::Group(group)));
}
}
match token {
Some(TokenTree::Group(group)) => {
let delimiter = group.delimiter();
let content = group.stream();
let span = group.span();
if delimiter == Delimiter::Bracket && is_paste_operation(&content) {
let segments = parse_bracket_as_segments(content, span)?;
let pasted = segment::paste(&segments)?;
let tokens = pasted_to_tokens(pasted, span)?;
expanded.extend(tokens);
*contains_paste = true;
} else if flatten_single_interpolation
&& delimiter == Delimiter::None
&& is_single_interpolation_group(&content)
{
expanded.extend(content);
*contains_paste = true;
} else {
let mut group_contains_paste = false;
let is_attribute = delimiter == Delimiter::Bracket
&& (lookbehind == Lookbehind::Pound || lookbehind == Lookbehind::PoundBang);
let mut nested = expand(
content,
&mut group_contains_paste,
flatten_single_interpolation && !is_attribute,
)?;
if is_attribute {
nested = expand_attr(nested, span, &mut group_contains_paste)?;
}
let group = if group_contains_paste {
let mut group = Group::new(delimiter, nested);
group.set_span(span);
*contains_paste = true;
group
} else {
group.clone()
};
if delimiter != Delimiter::None {
expanded.extend(iter::once(TokenTree::Group(group)));
} else if lookbehind == Lookbehind::DoubleColon {
expanded.extend(group.stream());
*contains_paste = true;
} else {
prev_none_group = Some(group);
}
}
lookbehind = Lookbehind::Other;
}
Some(TokenTree::Punct(punct)) => {
lookbehind = match punct.as_char() {
':' if lookbehind == Lookbehind::JointColon => Lookbehind::DoubleColon,
':' if punct.spacing() == Spacing::Joint => Lookbehind::JointColon,
'#' => Lookbehind::Pound,
'!' if lookbehind == Lookbehind::Pound => Lookbehind::PoundBang,
_ => Lookbehind::Other,
};
expanded.extend(iter::once(TokenTree::Punct(punct)));
}
Some(other) => {
lookbehind = Lookbehind::Other;
expanded.extend(iter::once(other));
}
None => return Ok(expanded),
}
}
}
#[derive(PartialEq)]
enum Lookbehind {
JointColon,
DoubleColon,
Pound,
PoundBang,
Other,
}
fn is_single_interpolation_group(input: &TokenStream) -> bool {
#[derive(PartialEq)]
enum State {
Init,
Ident,
Literal,
Apostrophe,
Lifetime,
Colon1,
Colon2,
}
let mut state = State::Init;
for tt in input.clone() {
state = match (state, &tt) {
(State::Init, TokenTree::Ident(_)) => State::Ident,
(State::Init, TokenTree::Literal(_)) => State::Literal,
(State::Init, TokenTree::Punct(punct)) if punct.as_char() == '\'' => State::Apostrophe,
(State::Apostrophe, TokenTree::Ident(_)) => State::Lifetime,
(State::Ident, TokenTree::Punct(punct))
if punct.as_char() == ':' && punct.spacing() == Spacing::Joint =>
{
State::Colon1
}
(State::Colon1, TokenTree::Punct(punct))
if punct.as_char() == ':' && punct.spacing() == Spacing::Alone =>
{
State::Colon2
}
(State::Colon2, TokenTree::Ident(_)) => State::Ident,
_ => return false,
};
}
state == State::Ident || state == State::Literal || state == State::Lifetime
}
fn is_paste_operation(input: &TokenStream) -> bool {
let mut tokens = input.clone().into_iter();
match &tokens.next() {
Some(TokenTree::Punct(punct)) if punct.as_char() == '<' => {}
_ => return false,
}
let mut has_token = false;
loop {
match &tokens.next() {
Some(TokenTree::Punct(punct)) if punct.as_char() == '>' => {
return has_token && tokens.next().is_none();
}
Some(_) => has_token = true,
None => return false,
}
}
}
#[cfg_attr(coverage_nightly, coverage(off))]
fn check_close_angle_token(token: Option<TokenTree>, scope: Span) -> Result<()> {
match token {
Some(TokenTree::Punct(punct)) if punct.as_char() == '>' => Ok(()),
Some(wrong) => Err(Error::new(wrong.span(), "expected `>`")),
None => Err(Error::new(scope, "expected `[< ... >]`")),
}
}
#[cfg_attr(coverage_nightly, coverage(off))]
fn try_convert_unicode_escape(value: &mut String) -> bool {
if value.starts_with("'\\u{") {
let hex = &value[4..value.len() - 2];
if let Ok(unsigned) = u32::from_str_radix(hex, 16) {
if let Some(ch) = char::from_u32(unsigned) {
value.clear();
value.push(ch);
return true;
}
}
}
false
}
fn parse_bracket_as_segments(input: TokenStream, scope: Span) -> Result<Vec<Segment>> {
let mut tokens = input.into_iter().peekable();
match &tokens.next() {
Some(TokenTree::Punct(punct)) if punct.as_char() == '<' => {}
Some(wrong) => return Err(Error::new(wrong.span(), "expected `<`")),
None => return Err(Error::new(scope, "expected `[< ... >]`")),
}
let mut segments = segment::parse(&mut tokens)?;
check_close_angle_token(tokens.next(), scope)?;
if let Some(unexpected) = tokens.next() {
return Err(Error::new(
unexpected.span(),
"unexpected input, expected `[< ... >]`",
));
}
for segment in &mut segments {
if let Segment::String(string) = segment {
if try_convert_unicode_escape(&mut string.value) {
continue;
}
if string.value.contains(&['\\', '.', '+'][..])
|| string.value.starts_with("b'")
|| string.value.starts_with("b\"")
|| string.value.starts_with("br\"")
{
return Err(Error::new(string.span, "unsupported literal"));
}
let mut range = 0..string.value.len();
if string.value.starts_with("r\"") {
range.start += 2;
range.end -= 1;
} else if string.value.starts_with(&['"', '\''][..]) {
range.start += 1;
range.end -= 1;
}
string.value = string.value[range].replace('-', "_");
}
}
Ok(segments)
}
fn pasted_to_tokens(mut pasted: String, span: Span) -> Result<TokenStream> {
let mut raw_mode = false;
let mut tokens = TokenStream::new();
if pasted.starts_with(|ch: char| ch.is_ascii_digit()) {
let literal = match panic::catch_unwind(|| Literal::from_str(&pasted)) {
Ok(Ok(literal)) => TokenTree::Literal(literal),
Ok(Err(LexError { .. })) | Err(_) => {
return Err(Error::new(
span,
&format!("`{:?}` is not a valid literal", pasted),
));
}
};
tokens.extend(iter::once(literal));
return Ok(tokens);
}
if pasted.starts_with('\'') {
let mut apostrophe = TokenTree::Punct(Punct::new('\'', Spacing::Joint));
apostrophe.set_span(span);
tokens.extend(iter::once(apostrophe));
pasted.remove(0);
}
if pasted.starts_with("r#") {
raw_mode = true;
}
let ident = match panic::catch_unwind(|| {
if raw_mode {
let mut spasted = pasted.clone();
spasted.remove(0);
spasted.remove(0);
Ident::new_raw(&spasted, span)
} else {
Ident::new(&pasted, span)
}
}) {
Ok(ident) => TokenTree::Ident(ident),
Err(_) => {
return Err(Error::new(
span,
&format!("`{:?}` is not a valid identifier", pasted),
));
}
};
tokens.extend(iter::once(ident));
Ok(tokens)
}
#[cfg(coverage_nightly)]
#[coverage(off)]
#[proc_macro]
#[doc(hidden)]
pub fn paste_test(input: TokenStream) -> TokenStream {
let scope = Span::call_site();
test_helpers::expand_attr_test(scope);
test_helpers::parse_bracket_as_segments_test(scope);
{
let mut multi_inner = TokenStream::new();
multi_inner.extend([TokenTree::Ident(Ident::new("a", scope))]);
multi_inner.extend([TokenTree::Ident(Ident::new("b", scope))]);
let multi_none = TokenTree::Group(Group::new(Delimiter::None, multi_inner));
let _ = segment::get_token_tree_string_value(&multi_none);
}
{
let empty_none = TokenTree::Group(Group::new(Delimiter::None, TokenStream::new()));
let _ = segment::get_token_tree_string_value(&empty_none);
}
let mut out = TokenStream::from_str(
"compile_error!(\"paste_test! is for internal coverage testing only and must not be called outside of test context\");",
)
.unwrap();
out.extend(input);
out
}
#[cfg(doctest)]
#[doc(hidden)]
mod doc_tests {
fn paste_test_macro_for_internal_coverage() {}
fn test_non_paste_bracket_returns_input() {}
fn test_flatten_single_ident_group() {}
fn test_flatten_lifetime_group() {}
fn test_flatten_path_group() {}
fn test_flatten_literal_group() {}
fn test_double_colon_none_group() {}
fn test_none_group_complex_type_before_double_colon() {}
fn test_inner_mod() {}
fn test_double_colon_none_group_in_attr() {}
fn test_char_unicode_escape_in_paste() {}
fn test_raw_string_in_paste() {}
fn test_lifetime_paste_tokens() {}
fn test_raw_mode_paste_tokens() {}
fn test_char_unicode_paste_tokens() {}
fn test_none_group_not_followed_by_double_colon() {}
fn test_error_invalid_literal_with_trailing_tokens() {}
fn test_error_invalid_identifier_special_char() {}
fn test_error_expected_bracket_with_angle() {}
fn test_error_expected_closing_angle() {}
fn test_error_unexpected_token_after_closing_angle() {}
fn test_error_unsupported_escaped_string() {}
fn test_error_unsupported_byte_string() {}
fn test_error_unsupported_byte_char_literal() {}
fn test_error_unsupported_raw_byte_string() {}
fn test_error_unsupported_string_with_dots() {}
fn test_error_unsupported_string_with_plus() {}
fn test_error_expected_content_in_brackets() {}
fn test_error_invalid_numeric_literal() {}
fn test_error_no_tokens_in_paste() {}
fn test_error_lifetime_without_identifier() {}
fn test_error_invalid_ident_starting_with_number() {}
fn test_error_terminated_string_literal() {}
fn test_error_multiple_tokens_non_paste() {}
fn test_error_paste_result_type_mismatch() {}
fn test_error_multiple_lifetimes() {}
fn test_error_multiple_numeric_tokens() {}
fn test_error_method_call_on_paste_result() {}
fn test_error_raw_keyword_identifier() {}
fn test_error_empty_macro_result() {}
fn test_error_hyphen_at_start() {}
fn test_error_apostrophe_with_number() {}
fn test_error_invalid_literal_argument() {}
fn test_error_invalid_unicode_escape() {}
fn test_error_incomplete_hex_literal() {}
fn test_error_incomplete_binary_literal() {}
fn test_error_range_literal_invalid() {}
fn test_error_keyword_as_identifier() {}
fn test_error_fn_keyword() {}
fn test_error_match_keyword() {}
fn test_error_brace_group_in_paste() {}
fn test_error_paren_group_in_paste() {}
fn test_error_double_ampersand() {}
fn test_error_double_pipe() {}
fn test_error_random_special_chars() {}
fn test_error_undefined_identifier_value() {}
fn test_error_invalid_paste_inside_brace_group() {}
fn test_error_pasted_incomplete_hex_literal() {}
fn test_error_attr_trailing_tokens_after_paren_group() {}
fn test_error_invalid_colon_sequence() {}
fn test_error_invalid_colon_patterns() {}
fn test_error_bracket_first_token_not_less_than() {}
fn test_error_bracket_closing_token_not_greater_than() {}
fn test_error_extra_tokens_after_bracket() {}
fn test_error_invalid_hex_unicode() {}
fn test_macro_group_flattening() {}
fn test_valid_paste_operation() {}
fn test_valid_path_with_double_colon() {}
fn test_valid_modifiers() {}
fn test_type_annotation() {}
}