moxy_token/
lib.rs

1#![cfg_attr(nightly, feature(proc_macro_diagnostic, proc_macro_span, proc_macro_def_site,))]
2
3//! # Moxy tokens
4//!
5//! Token types, lexing, spans, and compiler-token conversions for Rust syntax
6//! tooling. [`TokenStream`] is an owned sequence of [`TokenTree`] values that
7//! preserves token text, punctuation spacing, and source spans.
8//!
9//! ## Quick start
10//!
11//! Parse source into a token stream, or construct individual token types with
12//! the provided macros:
13//!
14//! ```ignore
15//! use moxy::token::{ident, TokenStream};
16//! use moxy::Token;
17//!
18//! let stream: TokenStream = "fn generated() {}".parse().unwrap();
19//! let name = ident!(generated);
20//! let semi: Token![;] = Default::default();
21//! assert_eq!(name.to_string(), "generated");
22//! assert_eq!(semi.to_string(), ";");
23//! ```
24//!
25//! ## Integrations
26//!
27//! Enable `serde` to serialize supported token types. Enable `proc-macro2` for
28//! conversions with `proc_macro2`;
29
30extern crate proc_macro;
31
32mod bridge;
33mod delim;
34mod group;
35mod ident;
36mod keyword;
37mod lex;
38mod lit;
39mod punct;
40/// Source files, locations, and source maps used by spans.
41pub mod source;
42mod spacing;
43mod span;
44mod stream;
45mod tree;
46
47#[doc(inline)]
48pub use delim::*;
49#[doc(inline)]
50pub use group::*;
51#[doc(inline)]
52pub use ident::*;
53#[doc(inline)]
54pub use keyword::*;
55#[doc(inline)]
56pub use lex::*;
57#[doc(inline)]
58pub use lit::*;
59#[doc(inline)]
60pub use punct::*;
61#[doc(inline)]
62pub use source::Location;
63#[doc(inline)]
64pub use spacing::*;
65#[doc(inline)]
66pub use span::{DelimSpan, RangeSpan, Span, Spanner};
67#[doc(inline)]
68pub use stream::*;
69#[doc(inline)]
70pub use tree::*;
71
72/// Emits a value as tokens into a destination token stream.
73/// A public Rust token API type for to tokens<t.
74pub trait ToTokens<T = TokenStream> {
75    fn to_tokens(&self, tokens: &mut T);
76}
77
78/// Convenience methods for values that can emit a [`TokenStream`].
79/// A public Rust token API type for to token stream:.
80pub trait ToTokenStream: ToTokens<TokenStream> {
81    fn to_token_stream(&self) -> TokenStream {
82        let mut tokens = TokenStream::new();
83        self.to_tokens(&mut tokens);
84        tokens
85    }
86
87    fn into_token_stream(self) -> TokenStream
88    where
89        Self: Sized,
90    {
91        self.to_token_stream()
92    }
93}
94
95/// Fallibly converts a parsing input into a [`TokenStream`].
96///
97/// Unlike [`ToTokenStream`], this trait is intended for input that may need
98/// lexing, such as Rust source text.
99pub trait TryIntoTokenStream: ToTokenStream {
100    fn try_into_token_stream(self) -> Result<TokenStream, LexError>;
101}
102
103impl<X: ToTokens<TokenStream> + ?Sized> ToTokenStream for X {}
104
105impl TryIntoTokenStream for TokenStream {
106    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
107        Ok(self)
108    }
109}
110
111impl TryIntoTokenStream for &TokenStream {
112    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
113        Ok(self.clone())
114    }
115}
116
117impl TryIntoTokenStream for &&TokenStream {
118    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
119        Ok((*self).clone())
120    }
121}
122
123impl TryIntoTokenStream for &str {
124    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
125        TokenStream::from_string(self.to_string())
126    }
127}
128
129impl TryIntoTokenStream for &&str {
130    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
131        TokenStream::from_string((*self).to_string())
132    }
133}
134
135impl TryIntoTokenStream for String {
136    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
137        TokenStream::from_string(self)
138    }
139}
140
141impl TryIntoTokenStream for &String {
142    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
143        TokenStream::from_string(self.clone())
144    }
145}
146
147impl TryIntoTokenStream for &&String {
148    fn try_into_token_stream(self) -> Result<TokenStream, LexError> {
149        TokenStream::from_string((*self).clone())
150    }
151}
152
153impl<T: ToTokens> ToTokens for ::std::boxed::Box<T> {
154    fn to_tokens(&self, tokens: &mut TokenStream) {
155        (**self).to_tokens(tokens);
156    }
157}
158
159impl<T: ToTokens> ToTokens for &T {
160    fn to_tokens(&self, tokens: &mut TokenStream) {
161        ToTokens::to_tokens(*self, tokens);
162    }
163}
164
165impl<T: ToTokens> ToTokens for Option<T> {
166    fn to_tokens(&self, tokens: &mut TokenStream) {
167        if let Some(v) = self {
168            v.to_tokens(tokens);
169        }
170    }
171}
172
173impl<T: ToTokens> ToTokens for Vec<T> {
174    fn to_tokens(&self, tokens: &mut TokenStream) {
175        for v in self {
176            v.to_tokens(tokens);
177        }
178    }
179}
180
181impl<T: ToTokens, E: ToTokens> ToTokens for Result<T, E> {
182    fn to_tokens(&self, tokens: &mut TokenStream) {
183        match self {
184            Self::Ok(v) => v.to_tokens(tokens),
185            Self::Err(err) => err.to_tokens(tokens),
186        }
187    }
188}
189
190impl<T: Spanner> Spanner for Option<T> {
191    fn span(&self) -> Span {
192        match self {
193            Self::None => Span::call_site(),
194            Self::Some(v) => v.span(),
195        }
196    }
197}
198
199impl<T: Spanner, E: Spanner> Spanner for Result<T, E> {
200    fn span(&self) -> Span {
201        match self {
202            Self::Err(v) => v.span(),
203            Self::Ok(v) => v.span(),
204        }
205    }
206}
207
208/// Map a Rust punctuation or keyword symbol to its [`crate`] token type.
209#[macro_export]
210macro_rules! Token {
211    // --- punctuation: single char ---
212    [&]     => { $crate::And };
213    [|]     => { $crate::Or };
214    [!]     => { $crate::Not };
215    [~]     => { $crate::Tilde };
216    [+]     => { $crate::Plus };
217    [-]     => { $crate::Minus };
218    [_]     => { $crate::Underscore };
219    [*]     => { $crate::Star };
220    [/]     => { $crate::Slash };
221    [%]     => { $crate::Percent };
222    [^]     => { $crate::Caret };
223    [=]     => { $crate::Eq };
224    [<]     => { $crate::Lt };
225    [>]     => { $crate::Gt };
226    [@]     => { $crate::At };
227    [.]     => { $crate::Dot };
228    [,]     => { $crate::Comma };
229    [;]     => { $crate::Semi };
230    [:]     => { $crate::Colon };
231    [#]     => { $crate::Pound };
232    [$]     => { $crate::Dollar };
233    [?]     => { $crate::Question };
234
235    // --- keywords ---
236    [as]          => { $crate::As };
237    [async]       => { $crate::Async };
238    [auto]        => { $crate::Auto };
239    [await]       => { $crate::Await };
240    [become]      => { $crate::Become };
241    [box]         => { $crate::Box };
242    [break]       => { $crate::Break };
243    [const]       => { $crate::Const };
244    [continue]    => { $crate::Continue };
245    [crate]       => { $crate::Crate };
246    [default]     => { $crate::Default };
247    [do]          => { $crate::Do };
248    [dyn]         => { $crate::Dyn };
249    [else]        => { $crate::Else };
250    [enum]        => { $crate::Enum };
251    [extern]      => { $crate::Extern };
252    [final]       => { $crate::Final };
253    [fn]          => { $crate::Fn };
254    [for]         => { $crate::For };
255    [if]          => { $crate::If };
256    [impl]        => { $crate::Impl };
257    [in]          => { $crate::In };
258    [let]         => { $crate::Let };
259    [loop]        => { $crate::Loop };
260    [macro]       => { $crate::Macro };
261    [macro_rules] => { $crate::MacroRules };
262    [match]       => { $crate::Match };
263    [mod]         => { $crate::Mod };
264    [move]        => { $crate::Move };
265    [mut]         => { $crate::Mut };
266    [override]    => { $crate::Override };
267    [priv]        => { $crate::Priv };
268    [pub]         => { $crate::Pub };
269    [raw]         => { $crate::Raw };
270    [ref]         => { $crate::Ref };
271    [return]      => { $crate::Return };
272    [safe]        => { $crate::Safe };
273    [Self]        => { $crate::SelfType };
274    [self]        => { $crate::SelfValue };
275    [static]      => { $crate::Static };
276    [struct]      => { $crate::Struct };
277    [super]       => { $crate::Super };
278    [trait]       => { $crate::Trait };
279    [try]         => { $crate::Try };
280    [type]        => { $crate::Type };
281    [typeof]      => { $crate::Typeof };
282    [union]       => { $crate::Union };
283    [unsafe]      => { $crate::Unsafe };
284    [unsized]     => { $crate::Unsized };
285    [use]         => { $crate::Use };
286    [virtual]     => { $crate::Virtual };
287    [where]       => { $crate::Where };
288    [while]       => { $crate::While };
289    [yield]       => { $crate::Yield };
290}
moxy_token/lib.rs

moxy_token/
lib.rs
