token_metadata/

lib.rs

#![doc = include_str!("../README.md")]

extern crate alloc;

use ::alloc::vec::Vec;
use ::core::{
	fmt::Debug,
	ops::{
		Deref,
		DerefMut,
	},
};
use ::proc_macro2::{
	Delimiter,
	Group,
	Ident,
	Literal,
	Punct,
	Span,
	TokenStream,
	TokenTree,
	extra::DelimSpan,
};

/// A version of [`Group`] with arbitrary attached metadata. The tokens within
/// this [`GroupWithMetadata`] also have metadata.
///
/// The fields of this struct are public, unlike [`Group`]'s. There are no
/// getter/setter methods: use the fields directly.
///
///  See [`proc_macro2`]'s documentation for more information on [`Group`].
#[derive(Clone, Debug)]
pub struct GroupWithMetadata<G, I = G, P = G, L = I> {
	/// See [`Group::delimiter`].
	pub delimiter: Delimiter,
	/// A vector holding the tokens within this group, each with their own
	/// metadata.
	pub contents: TokenWithMetadataVec<G, I, P, L>,
	/// See [`Group::span`].
	/// This span covers the entire group, including the delimiters.
	pub span: Span,
	/// The associated metadata.
	pub metadata: G,
}

impl<G, I, P, L> GroupWithMetadata<G, I, P, L> {
	/// Strips the metadata from this [`GroupWithMetadata`], returning an
	/// ordinary [`Group`].
	///
	/// The tokens within the group will also be stripped of their metadata.
	#[must_use = "If you meant to drop this value, use `std::mem::drop` instead."]
	pub fn strip_metadata(self) -> Group {
		let mut group = Group::new(
			self.delimiter,
			token_with_metadata_iter_ext::iter_tokens_strip_metadata(self.contents).collect(),
		);
		group.set_span(self.span);
		group
	}

	/// See [`Group::delimiter`].
	#[inline(always)]
	#[deprecated(note = "Use the `delimiter` field directly.")]
	pub fn delimiter(&self) -> Delimiter {
		self.delimiter
	}

	/// See [`Group::span`].
	#[inline(always)]
	#[must_use]
	#[deprecated(note = "Use the `span` field directly.")]
	pub fn span(&self) -> Span {
		self.span
	}

	/// See [`Group::span_open`].
	#[must_use]
	pub fn span_open(&self) -> Span {
		// Unfortunately, proc-macro2 does not expose span_open/spam_close methods.
		// We need to create a dummy Group to access them.
		let mut dummy = Group::new(self.delimiter, TokenStream::new());
		dummy.set_span(self.span);
		dummy.span_open()
	}

	/// See [`Group::span_close`].
	#[must_use]
	pub fn span_close(&self) -> Span {
		let mut dummy = Group::new(self.delimiter, TokenStream::new());
		dummy.set_span(self.span);
		dummy.span_close()
	}

	/// See [`Group::delim_span`].
	#[must_use]
	pub fn delim_span(&self) -> DelimSpan {
		let mut dummy = Group::new(self.delimiter, TokenStream::new());
		dummy.set_span(self.span);
		dummy.delim_span()
	}

	/// See [`Group::set_span`].
	#[inline(always)]
	#[deprecated(note = "Use the `span` field directly.")]
	pub fn set_span(&mut self, span: Span) {
		self.span = span;
	}
}

impl<G, I, P, L> From<Group> for GroupWithMetadata<G, I, P, L>
where
	G: Default,
	I: Default,
	P: Default,
	L: Default,
{
	fn from(value: Group) -> Self {
		Self {
			delimiter: value.delimiter(),
			contents: token_with_metadata_iter_ext::iter_tokens_add_default_metadata(value.stream()).collect(),
			span: value.span(),
			metadata: G::default(),
		}
	}
}

impl<T> From<(Group, &T)> for GroupWithMetadata<T>
where
	T: Clone,
{
	fn from((value, metadata): (Group, &T)) -> Self {
		Self {
			delimiter: value.delimiter(),
			contents: token_with_metadata_iter_ext::iter_tokens_add_metadata(value.stream(), metadata).collect(),
			span: value.span(),
			metadata: metadata.clone(),
		}
	}
}

impl<G, I, P, L> From<(Group, &G, &I, &P, &L)> for GroupWithMetadata<G, I, P, L>
where
	G: Clone,
	I: Clone,
	P: Clone,
	L: Clone,
{
	fn from((value, g, i, p, l): (Group, &G, &I, &P, &L)) -> Self {
		Self {
			delimiter: value.delimiter(),
			contents: token_with_metadata_iter_ext::iter_tokens_add_specific_metadata(value.stream(), g, i, p, l).collect(),
			span: value.span(),
			metadata: g.clone(),
		}
	}
}

/// A simple struct holding a token and its associated metadata.
///
/// Though [`TokenWithMetadata::token`] is generic, it should be a token from
/// [`proc_macro2`] other than [`Group`] ([`Ident`], [`Punct`], or [`Literal`]).
///
/// To get the token without metadata, simply read the
/// [`TokenWithMetadata::token`] field.
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
pub struct TokenWithMetadata<T, U> {
	/// The token.
	pub token: T,
	/// The associated metadata.
	pub metadata: U,
}

/// Allow accessing the token's methods directly.
impl<T, U> Deref for TokenWithMetadata<T, U> {
	type Target = T;

	#[inline(always)]
	fn deref(&self) -> &Self::Target {
		&self.token
	}
}

/// Allow accessing the token's methods directly.
impl<T, U> DerefMut for TokenWithMetadata<T, U> {
	#[inline(always)]
	fn deref_mut(&mut self) -> &mut Self::Target {
		&mut self.token
	}
}

impl<T, U: Default> From<T> for TokenWithMetadata<T, U> {
	#[inline(always)]
	fn from(token: T) -> Self {
		Self { token, metadata: U::default() }
	}
}

impl<T, U> From<(T, U)> for TokenWithMetadata<T, U> {
	#[inline(always)]
	fn from((token, metadata): (T, U)) -> Self {
		Self { token, metadata }
	}
}

/// A version of [`TokenTree`] with arbitrary attached metadata.
///
/// Each variant holds metadata of a different type. The types are:
/// - `G`: For [`GroupWithMetadata`]s (in [`proc_macro2`], [`Group`]s).
/// - `I`: For [`Ident`] tokens (defaults to the same type as `G`).
/// - `P`: For [`Punct`] tokens (defaults to the same type as `G`).
/// - `L`: For [`Literal`] tokens (defaults to the same type as `I`).
///
/// See [`proc_macro2`]'s documentation for more information.
#[derive(Clone, Debug)]
pub enum TokenTreeWithMetadata<G, I = G, P = G, L = I> {
	/// A [`GroupWithMetadata`] token with its corresponding metadata.
	/// The tokens within the group, correspondingly, also have metadata, with
	/// the same types.
	Group(GroupWithMetadata<G, I, P, L>),
	/// An [`Ident`] token with its corresponding metadata.
	Ident(TokenWithMetadata<Ident, I>),
	/// A [`Punct`] token with its corresponding metadata.
	Punct(TokenWithMetadata<Punct, P>),
	/// A [`Literal`] token with its corresponding metadata.
	Literal(TokenWithMetadata<Literal, L>),
}

impl<G, I, P, L> TokenTreeWithMetadata<G, I, P, L> {
	/// Strips the metadata from this [`TokenTreeWithMetadata`], returning an
	/// ordinary [`TokenTree`]. Child [`GroupWithMetadata`]s will also be
	/// stripped of their metadata.
	#[inline]
	#[must_use = "If you meant to drop this value, use `std::mem::drop` instead."]
	pub fn strip_metadata(self) -> TokenTree {
		match self {
			| TokenTreeWithMetadata::Group(group) => TokenTree::Group(group.strip_metadata()),
			| TokenTreeWithMetadata::Ident(twm) => TokenTree::Ident(twm.token),
			| TokenTreeWithMetadata::Punct(twm) => TokenTree::Punct(twm.token),
			| TokenTreeWithMetadata::Literal(twm) => TokenTree::Literal(twm.token),
		}
	}

	/// Gets the span of this token.
	///
	/// See [`TokenTree::span`].
	#[inline]
	#[must_use]
	pub fn span(&self) -> Span {
		match self {
			| TokenTreeWithMetadata::Group(group) => group.span,
			| TokenTreeWithMetadata::Ident(twm) => twm.token.span(),
			| TokenTreeWithMetadata::Punct(twm) => twm.token.span(),
			| TokenTreeWithMetadata::Literal(twm) => twm.token.span(),
		}
	}

	/// Sets the span of this token.
	///
	/// See [`TokenTree::set_span`].
	#[inline]
	pub fn set_span(&mut self, span: Span) {
		match self {
			| TokenTreeWithMetadata::Group(group) => group.span = span,
			| TokenTreeWithMetadata::Ident(twm) => twm.token.set_span(span),
			| TokenTreeWithMetadata::Punct(twm) => twm.token.set_span(span),
			| TokenTreeWithMetadata::Literal(twm) => twm.token.set_span(span),
		}
	}
}

impl<T> TokenTreeWithMetadata<T> {
	/// Consumes self and returns the metadata associated with this token.
	/// Only available when all metadata types are the same.
	#[inline]
	#[must_use = "If you meant to drop this value, use `std::mem::drop` instead."]
	pub fn get_metadata(self) -> T {
		match self {
			| Self::Group(group) => group.metadata,
			| Self::Ident(twm) => twm.metadata,
			| Self::Punct(twm) => twm.metadata,
			| Self::Literal(twm) => twm.metadata,
		}
	}

	/// Returns a reference to the metadata associated with this token.
	/// Only available when all metadata types are the same.
	#[inline]
	#[must_use]
	pub fn get_metadata_ref(&self) -> &T {
		match self {
			| Self::Group(group) => &group.metadata,
			| Self::Ident(twm) => &twm.metadata,
			| Self::Punct(twm) => &twm.metadata,
			| Self::Literal(twm) => &twm.metadata,
		}
	}

	/// Returns a mutable reference to the metadata associated with this token.
	/// Only available when all metadata types are the same.
	#[inline]
	#[must_use]
	pub fn get_metadata_mut(&mut self) -> &mut T {
		match self {
			| Self::Group(group) => &mut group.metadata,
			| Self::Ident(twm) => &mut twm.metadata,
			| Self::Punct(twm) => &mut twm.metadata,
			| Self::Literal(twm) => &mut twm.metadata,
		}
	}

	/// Sets the metadata associated with this token.
	/// Only available when all metadata types are the same.
	#[inline]
	pub fn set_metadata(&mut self, metadata: T) {
		match self {
			| Self::Group(group) => group.metadata = metadata,
			| Self::Ident(twm) => twm.metadata = metadata,
			| Self::Punct(twm) => twm.metadata = metadata,
			| Self::Literal(twm) => twm.metadata = metadata,
		}
	}
}

impl<G, I, P, L> From<TokenTree> for TokenTreeWithMetadata<G, I, P, L>
where
	G: Default,
	I: Default,
	P: Default,
	L: Default,
{
	#[inline]
	fn from(tree: TokenTree) -> Self {
		match tree {
			| TokenTree::Group(group) => Self::Group(group.into()),
			| TokenTree::Ident(ident) => Self::Ident(ident.into()),
			| TokenTree::Punct(punct) => Self::Punct(punct.into()),
			| TokenTree::Literal(literal) => Self::Literal(literal.into()),
		}
	}
}

impl<T> From<(TokenTree, &T)> for TokenTreeWithMetadata<T>
where
	T: Clone,
{
	#[inline]
	fn from((tree, metadata): (TokenTree, &T)) -> Self {
		match tree {
			| TokenTree::Group(group) => Self::Group((group, metadata).into()),
			| TokenTree::Ident(ident) => Self::Ident((ident, metadata.clone()).into()),
			| TokenTree::Punct(punct) => Self::Punct((punct, metadata.clone()).into()),
			| TokenTree::Literal(literal) => Self::Literal((literal, metadata.clone()).into()),
		}
	}
}

impl<G, I, P, L> From<(TokenTree, &G, &I, &P, &L)> for TokenTreeWithMetadata<G, I, P, L>
where
	G: Clone,
	I: Clone,
	P: Clone,
	L: Clone,
{
	#[inline]
	fn from((tree, g, i, p, l): (TokenTree, &G, &I, &P, &L)) -> Self {
		match tree {
			| TokenTree::Group(group) => Self::Group((group, g, i, p, l).into()),
			| TokenTree::Ident(ident) => Self::Ident((ident, i.clone()).into()),
			| TokenTree::Punct(punct) => Self::Punct((punct, p.clone()).into()),
			| TokenTree::Literal(literal) => Self::Literal((literal, l.clone()).into()),
		}
	}
}

impl<G, I, P, L> From<Group> for TokenTreeWithMetadata<G, I, P, L>
where
	G: Default,
	I: Default,
	P: Default,
	L: Default,
{
	#[inline(always)]
	fn from(group: Group) -> Self {
		Self::Group(group.into())
	}
}

impl<T> From<(Group, &T)> for TokenTreeWithMetadata<T>
where
	T: Clone,
{
	#[inline(always)]
	fn from(group_and_val: (Group, &T)) -> Self {
		Self::Group(group_and_val.into())
	}
}

impl<G, I, P, L> From<(Group, &G, &I, &P, &L)> for TokenTreeWithMetadata<G, I, P, L>
where
	G: Clone,
	I: Clone,
	P: Clone,
	L: Clone,
{
	#[inline(always)]
	fn from(group_and_vals: (Group, &G, &I, &P, &L)) -> Self {
		Self::Group(group_and_vals.into())
	}
}

impl<G, I, P, L> From<Ident> for TokenTreeWithMetadata<G, I, P, L>
where
	I: Default,
{
	#[inline(always)]
	fn from(ident: Ident) -> Self {
		Self::Ident(ident.into())
	}
}

impl<G, I, P, L> From<(Ident, I)> for TokenTreeWithMetadata<G, I, P, L> {
	#[inline(always)]
	fn from((ident, metadata): (Ident, I)) -> Self {
		Self::Ident((ident, metadata).into())
	}
}

impl<G, I, P, L> From<Punct> for TokenTreeWithMetadata<G, I, P, L>
where
	P: Default,
{
	#[inline(always)]
	fn from(punct: Punct) -> Self {
		Self::Punct(punct.into())
	}
}

impl<G, I, P, L> From<(Punct, P)> for TokenTreeWithMetadata<G, I, P, L> {
	#[inline(always)]
	fn from((punct, metadata): (Punct, P)) -> Self {
		Self::Punct((punct, metadata).into())
	}
}

impl<G, I, P, L> From<Literal> for TokenTreeWithMetadata<G, I, P, L>
where
	L: Default,
{
	#[inline(always)]
	fn from(literal: Literal) -> Self {
		Self::Literal(literal.into())
	}
}

impl<G, I, P, L> From<(Literal, L)> for TokenTreeWithMetadata<G, I, P, L> {
	#[inline(always)]
	fn from((literal, metadata): (Literal, L)) -> Self {
		Self::Literal((literal, metadata).into())
	}
}

/// A type alias for a [`Vec`] of [`TokenTreeWithMetadata`]s.
///
/// Since [`TokenTreeWithMetadata`]s are not obtained from a [`TokenStream`]
/// directly, and are built manually or derived from obtained [`TokenTree`]s, it
/// doesn't make much sense to use a restrictive "token with metadata
/// stream" type. Instead, use vectors of tokens with metadata.
pub type TokenWithMetadataVec<G, I = G, P = G, L = I> = Vec<TokenTreeWithMetadata<G, I, P, L>>;

/// Extension module for [`TokenWithMetadataVec`], or more generally, for
/// iterators of [`TokenTreeWithMetadata`]s.
pub mod token_with_metadata_iter_ext {
	use super::*;

	/// Strips the metadata from an iterator of [`TokenTreeWithMetadata`]s,
	/// returning an iterator of ordinary [`TokenTree`]s.
	pub fn iter_tokens_strip_metadata<G, I, P, L>(iter: impl IntoIterator<Item = TokenTreeWithMetadata<G, I, P, L>>) -> impl Iterator<Item = TokenTree> {
		iter.into_iter().map(|ttwm| ttwm.strip_metadata())
	}

	/// Converts an iterator of ordinary [`TokenTree`]s into an iterator of
	/// [`TokenTreeWithMetadata`]s with default-initialized metadata.
	pub fn iter_tokens_add_default_metadata<G, I, P, L>(iter: impl IntoIterator<Item = TokenTree>) -> impl Iterator<Item = TokenTreeWithMetadata<G, I, P, L>>
	where
		G: Default,
		I: Default,
		P: Default,
		L: Default,
	{
		iter.into_iter().map(|tt| tt.into())
	}

	/// Converts an iterator of ordinary [`TokenTree`]s into an iterator of
	/// [`TokenTreeWithMetadata`]s with the given metadata.
	pub fn iter_tokens_add_metadata<T>(iter: impl IntoIterator<Item = TokenTree>, metadata: &T) -> impl Iterator<Item = TokenTreeWithMetadata<T>>
	where
		T: Clone,
	{
		iter.into_iter().map(move |tt| (tt, metadata).into())
	}

	/// Converts an iterator of ordinary [`TokenTree`]s into an iterator of
	/// [`TokenTreeWithMetadata`]s with the given specific metadata for each
	/// token type.
	pub fn iter_tokens_add_specific_metadata<G, I, P, L>(
		iter: impl IntoIterator<Item = TokenTree>,
		group: &G,
		ident: &I,
		punct: &P,
		literal: &L,
	) -> impl Iterator<Item = TokenTreeWithMetadata<G, I, P, L>>
	where
		G: Clone,
		I: Clone,
		P: Clone,
		L: Clone,
	{
		iter.into_iter().map(move |tt| (tt, group, ident, punct, literal).into())
	}
}