Skip to main content

moxy_token/
lib.rs

1#![cfg_attr(nightly, feature(proc_macro_diagnostic, proc_macro_span, proc_macro_def_site,))]
2
3//! # Moxy tokens
4//!
5//! Token types, lexing, spans, and compiler-token conversions for Rust syntax
6//! tooling. [`TokenStream`] is an owned sequence of [`TokenTree`] values that
7//! preserves token text, punctuation spacing, and source spans.
8//!
9//! ## Quick start
10//!
11//! Parse source into a token stream, or construct individual token types with
12//! the provided macros:
13//!
14//! ```ignore
15//! use moxy::token::{ident, TokenStream};
16//! use moxy::Token;
17//!
18//! let stream: TokenStream = "fn generated() {}".parse().unwrap();
19//! let name = ident!(generated);
20//! let semi: Token![;] = Default::default();
21//! assert_eq!(name.to_string(), "generated");
22//! assert_eq!(semi.to_string(), ";");
23//! ```
24//!
25//! ## Integrations
26//!
27//! Enable `serde` to serialize supported token types. Enable `proc-macro2` for
28//! conversions with `proc_macro2`;
29
30extern crate proc_macro;
31
32mod bridge;
33mod delim;
34mod group;
35mod ident;
36mod keyword;
37mod lex;
38mod lit;
39mod punct;
40#[cfg(feature = "quote")]
41mod quote;
42/// Source files, locations, and source maps used by spans.
43pub mod source;
44mod spacing;
45mod span;
46mod stream;
47mod tree;
48
49#[doc(inline)]
50pub use delim::*;
51#[doc(inline)]
52pub use group::*;
53#[doc(inline)]
54pub use ident::*;
55#[doc(inline)]
56pub use keyword::*;
57#[doc(inline)]
58pub use lex::*;
59#[doc(inline)]
60pub use lit::*;
61#[doc(inline)]
62pub use punct::*;
63#[cfg(feature = "quote")]
64#[doc(inline)]
65pub use quote::*;
66#[doc(inline)]
67pub use source::Location;
68#[doc(inline)]
69pub use spacing::*;
70#[doc(inline)]
71pub use span::{DelimSpan, RangeSpan, Span, Spanner};
72#[doc(inline)]
73pub use stream::*;
74#[doc(inline)]
75pub use tree::*;
76
77/// Emits a value as tokens into a destination token stream.
78/// A public Rust token API type for to tokens<t.
79pub trait ToTokens<T = TokenStream> {
80    fn to_tokens(&self, tokens: &mut T);
81}
82
83/// Convenience methods for values that can emit a [`TokenStream`].
84/// A public Rust token API type for to token stream:.
85pub trait ToTokenStream: ToTokens<TokenStream> {
86    fn to_token_stream(&self) -> TokenStream {
87        let mut tokens = TokenStream::new();
88        self.to_tokens(&mut tokens);
89        tokens
90    }
91
92    fn into_token_stream(self) -> TokenStream
93    where
94        Self: Sized,
95    {
96        self.to_token_stream()
97    }
98}
99
100/// Fallibly converts a parsing input into a [`TokenStream`].
101///
102/// Unlike [`ToTokenStream`], this trait is intended for input that may need
103/// lexing, such as Rust source text.
104pub trait TryIntoTokenStream: ToTokenStream {
105    fn try_into_token_stream(self) -> Result<TokenStream, LexError>;
106}
107
108impl<X: ToTokens<TokenStream> + ?Sized> ToTokenStream for X {}
109
110impl TryIntoTokenStream for TokenStream {
111    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
112        Ok(self)
113    }
114}
115
116impl TryIntoTokenStream for &TokenStream {
117    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
118        Ok(self.clone())
119    }
120}
121
122impl TryIntoTokenStream for &&TokenStream {
123    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
124        Ok((*self).clone())
125    }
126}
127
128impl TryIntoTokenStream for &str {
129    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
130        TokenStream::from_string(self.to_string())
131    }
132}
133
134impl TryIntoTokenStream for &&str {
135    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
136        TokenStream::from_string((*self).to_string())
137    }
138}
139
140impl TryIntoTokenStream for String {
141    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
142        TokenStream::from_string(self)
143    }
144}
145
146impl TryIntoTokenStream for &String {
147    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
148        TokenStream::from_string(self.clone())
149    }
150}
151
152impl TryIntoTokenStream for &&String {
153    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
154        TokenStream::from_string((*self).clone())
155    }
156}
157
158impl<T: ToTokens + ?Sized> ToTokens for ::std::boxed::Box<T> {
159    fn to_tokens(&self, tokens: &mut TokenStream) {
160        (**self).to_tokens(tokens);
161    }
162}
163
164impl<T: ToTokens + ?Sized> ToTokens for &T {
165    fn to_tokens(&self, tokens: &mut TokenStream) {
166        ToTokens::to_tokens(*self, tokens);
167    }
168}
169
170impl<T: ToTokens> ToTokens for Option<T> {
171    fn to_tokens(&self, tokens: &mut TokenStream) {
172        if let Some(v) = self {
173            v.to_tokens(tokens);
174        }
175    }
176}
177
178impl<T: ToTokens> ToTokens for Vec<T> {
179    fn to_tokens(&self, tokens: &mut TokenStream) {
180        for v in self {
181            v.to_tokens(tokens);
182        }
183    }
184}
185
186impl<T: ToTokens, E: ToTokens> ToTokens for Result<T, E> {
187    fn to_tokens(&self, tokens: &mut TokenStream) {
188        match self {
189            Self::Ok(v) => v.to_tokens(tokens),
190            Self::Err(err) => err.to_tokens(tokens),
191        }
192    }
193}
194
195impl<T: Spanner> Spanner for Option<T> {
196    fn span(&self) -> Span {
197        match self {
198            Self::None => Span::call_site(),
199            Self::Some(v) => v.span(),
200        }
201    }
202}
203
204impl<T: Spanner, E: Spanner> Spanner for Result<T, E> {
205    fn span(&self) -> Span {
206        match self {
207            Self::Err(v) => v.span(),
208            Self::Ok(v) => v.span(),
209        }
210    }
211}
212
213/// Map a Rust punctuation or keyword symbol to its [`crate`] token type.
214#[macro_export]
215macro_rules! Token {
216    // --- punctuation: single char ---
217    [&]     => { $crate::And };
218    [|]     => { $crate::Or };
219    [!]     => { $crate::Not };
220    [~]     => { $crate::Tilde };
221    [+]     => { $crate::Plus };
222    [-]     => { $crate::Minus };
223    [_]     => { $crate::Underscore };
224    [*]     => { $crate::Star };
225    [/]     => { $crate::Slash };
226    [%]     => { $crate::Percent };
227    [^]     => { $crate::Caret };
228    [=]     => { $crate::Eq };
229    [<]     => { $crate::Lt };
230    [>]     => { $crate::Gt };
231    [@]     => { $crate::At };
232    [.]     => { $crate::Dot };
233    [,]     => { $crate::Comma };
234    [;]     => { $crate::Semi };
235    [:]     => { $crate::Colon };
236    [#]     => { $crate::Pound };
237    [$]     => { $crate::Dollar };
238    [?]     => { $crate::Question };
239
240    // --- keywords ---
241    [as]          => { $crate::As };
242    [async]       => { $crate::Async };
243    [auto]        => { $crate::Auto };
244    [await]       => { $crate::Await };
245    [become]      => { $crate::Become };
246    [box]         => { $crate::Box };
247    [break]       => { $crate::Break };
248    [const]       => { $crate::Const };
249    [continue]    => { $crate::Continue };
250    [crate]       => { $crate::Crate };
251    [default]     => { $crate::Default };
252    [do]          => { $crate::Do };
253    [dyn]         => { $crate::Dyn };
254    [else]        => { $crate::Else };
255    [enum]        => { $crate::Enum };
256    [extern]      => { $crate::Extern };
257    [final]       => { $crate::Final };
258    [fn]          => { $crate::Fn };
259    [for]         => { $crate::For };
260    [if]          => { $crate::If };
261    [impl]        => { $crate::Impl };
262    [in]          => { $crate::In };
263    [let]         => { $crate::Let };
264    [loop]        => { $crate::Loop };
265    [macro]       => { $crate::Macro };
266    [macro_rules] => { $crate::MacroRules };
267    [match]       => { $crate::Match };
268    [mod]         => { $crate::Mod };
269    [move]        => { $crate::Move };
270    [mut]         => { $crate::Mut };
271    [override]    => { $crate::Override };
272    [priv]        => { $crate::Priv };
273    [pub]         => { $crate::Pub };
274    [raw]         => { $crate::Raw };
275    [ref]         => { $crate::Ref };
276    [return]      => { $crate::Return };
277    [safe]        => { $crate::Safe };
278    [Self]        => { $crate::SelfType };
279    [self]        => { $crate::SelfValue };
280    [static]      => { $crate::Static };
281    [struct]      => { $crate::Struct };
282    [super]       => { $crate::Super };
283    [trait]       => { $crate::Trait };
284    [try]         => { $crate::Try };
285    [type]        => { $crate::Type };
286    [typeof]      => { $crate::Typeof };
287    [union]       => { $crate::Union };
288    [unsafe]      => { $crate::Unsafe };
289    [unsized]     => { $crate::Unsized };
290    [use]         => { $crate::Use };
291    [virtual]     => { $crate::Virtual };
292    [where]       => { $crate::Where };
293    [while]       => { $crate::While };
294    [yield]       => { $crate::Yield };
295}