rustidy_ast/
token.rs

1//! Tokens
2
3// Imports
4use {
5	super::lifetime::LifetimeToken,
6	rustidy_ast_literal::{
7		ByteLiteral,
8		ByteStringLiteral,
9		CStringLiteral,
10		CharLiteral,
11		FloatLiteral,
12		IntegerLiteral,
13		RawByteStringLiteral,
14		RawCStringLiteral,
15		RawStringLiteral,
16		StringLiteral,
17	},
18	rustidy_ast_util::{IdentifierOrKeyword, RawIdentifier},
19	rustidy_format::{Format, Formattable},
20	rustidy_parse::{Parse, ParseError, ParserError, ParserTag},
21	rustidy_print::Print,
22	rustidy_util::{StrPopFirst, Whitespace},
23};
24
25// Exports
26pub use rustidy_ast_tokens::*;
27
28/// `Token`
29#[derive(PartialEq, Eq, Clone, Debug)]
30#[derive(serde::Serialize, serde::Deserialize)]
31#[derive(Parse, Formattable, Format, Print)]
32pub enum Token {
33	RawIdent(RawIdentifier),
34	CharLiteral(CharLiteral),
35	StringLiteral(StringLiteral),
36	RawStringLiteral(RawStringLiteral),
37	ByteLiteral(ByteLiteral),
38	ByteStringLiteral(ByteStringLiteral),
39	RawByteStringLiteral(RawByteStringLiteral),
40	CStringLiteral(CStringLiteral),
41	RawCStringLiteral(RawCStringLiteral),
42	IntegerLiteral(IntegerLiteral),
43	FloatLiteral(FloatLiteral),
44	LifetimeToken(LifetimeToken),
45	Punctuation(Punctuation),
46
47	IdentOrKeyword(IdentifierOrKeyword),
48	// TODO: Reserved tokens? Should we care as a formatter?
49}
50
51/// `Punctuation`
52#[derive(PartialEq, Eq, Clone, Debug)]
53#[derive(serde::Serialize, serde::Deserialize)]
54#[derive(Formattable, Format, Print)]
55pub enum Punctuation {
56	Eq(Eq),
57	Lt(Lt),
58	Le(Le),
59	EqEq(EqEq),
60	Ne(Ne),
61	Ge(Ge),
62	Gt(Gt),
63	AndAnd(AndAnd),
64	OrOr(OrOr),
65	Not(Not),
66	Tilde(Tilde),
67	Plus(Plus),
68	Minus(Minus),
69	Star(Star),
70	Slash(Slash),
71	Percent(Percent),
72	Caret(Caret),
73	And(And),
74	Or(Or),
75	Shl(Shl),
76	Shr(Shr),
77	PlusEq(PlusEq),
78	MinusEq(MinusEq),
79	StarEq(StarEq),
80	SlashEq(SlashEq),
81	PercentEq(PercentEq),
82	CaretEq(CaretEq),
83	AndEq(AndEq),
84	OrEq(OrEq),
85	ShlEq(ShlEq),
86	ShrEq(ShrEq),
87	At(At),
88	Dot(Dot),
89	DotDot(DotDot),
90	DotDotDot(DotDotDot),
91	DotDotEq(DotDotEq),
92	Comma(Comma),
93	Semi(Semi),
94	Colon(Colon),
95	PathSep(PathSep),
96	RArrow(RArrow),
97	LArrow(LArrow),
98	FatArrow(FatArrow),
99	Pound(Pound),
100	Dollar(Dollar),
101	Question(Question),
102	Underscore(Underscore),
103	ParenOpen(ParenOpen),
104	ParenClose(ParenClose),
105	BracketOpen(BracketOpen),
106	BracketClose(BracketClose),
107	BracesOpen(BracesOpen),
108	BracesClose(BracesClose),
109}
110
111// TODO: Autogenerate this impl?
112impl rustidy_parse::Parse for Punctuation {
113	type Error = PunctuationError;
114
115	fn parse_from(parser: &mut rustidy_parse::Parser) -> Result<Self, Self::Error> {
116		// TODO: Autogenerate this from `Punctuation`.
117		#[derive(Clone, Copy, Debug)]
118		enum Punct {
119			Eq,
120			Lt,
121			Le,
122			EqEq,
123			Ne,
124			Ge,
125			Gt,
126			AndAnd,
127			OrOr,
128			Not,
129			Tilde,
130			Plus,
131			Minus,
132			Star,
133			Slash,
134			Percent,
135			Caret,
136			And,
137			Or,
138			Shl,
139			Shr,
140			PlusEq,
141			MinusEq,
142			StarEq,
143			SlashEq,
144			PercentEq,
145			CaretEq,
146			AndEq,
147			OrEq,
148			ShlEq,
149			ShrEq,
150			At,
151			Dot,
152			DotDot,
153			DotDotDot,
154			DotDotEq,
155			Comma,
156			Semi,
157			Colon,
158			PathSep,
159			RArrow,
160			LArrow,
161			FatArrow,
162			Pound,
163			Dollar,
164			Question,
165			Underscore,
166			ParenOpen,
167			ParenClose,
168			BracketOpen,
169			BracketClose,
170			BracesOpen,
171			BracesClose,
172		}
173
174		let skip_plus = parser.has_tag(ParserTag::SkipTokenPlus);
175		let skip_star = parser.has_tag(ParserTag::SkipTokenStar);
176		let skip_dollar = parser.has_tag(ParserTag::SkipTokenDollar);
177		let skip_question = parser.has_tag(ParserTag::SkipTokenQuestion);
178		let skip_delimiters = parser.has_tag(ParserTag::SkipDelimiters);
179
180		let ws = parser.parse()?;
181		let res = parser.try_update_with(|s| {
182			let original_s = *s;
183			macro punct(
184				$len:literal, $punct:ident
185			) {
186				{
187				*s = &original_s[$len..];
188				Some(Punct::$punct)
189			}
190			}
191			match s.pop_first()? {
192				'=' => match s.pop_first() {
193					Some('=') => punct!(2, EqEq),
194					Some('>') => punct!(2, FatArrow),
195					_ => punct!(1, Eq),
196				},
197				'<' => match s.pop_first() {
198					Some('=') => punct!(2, Le),
199					Some('-') => punct!(2, LArrow),
200					Some('<') => match s.pop_first() {
201						Some('=') => punct!(3, ShlEq),
202						_ => punct!(2, Shl),
203					},
204					_ => punct!(1, Lt),
205				},
206				'>' => match s.pop_first() {
207					Some('=') => punct!(2, Ge),
208					Some('>') => match s.pop_first() {
209						Some('=') => punct!(3, ShrEq),
210						_ => punct!(2, Shr),
211					},
212					_ => punct!(1, Gt),
213				},
214				'!' => match s.pop_first() {
215					Some('=') => punct!(2, Ne),
216					_ => punct!(1, Not),
217				},
218				'&' => match s.pop_first() {
219					Some('&') => punct!(2, AndAnd),
220					Some('=') => punct!(2, AndEq),
221					_ => punct!(1, And),
222				},
223				'|' => match s.pop_first() {
224					Some('|') => punct!(2, OrOr),
225					Some('=') => punct!(2, OrEq),
226					_ => punct!(1, Or),
227				},
228				'+' => match s.pop_first() {
229					Some('=') => punct!(2, PlusEq),
230					_ if !skip_plus => punct!(1, Plus),
231					_ => None,
232				},
233				'-' => match s.pop_first() {
234					Some('=') => punct!(2, MinusEq),
235					Some('>') => punct!(2, RArrow),
236					_ => punct!(1, Minus),
237				},
238				'*' => match s.pop_first() {
239					Some('=') => punct!(2, StarEq),
240					_ if !skip_star => punct!(1, Star),
241					_ => None,
242				},
243				'/' => match s.pop_first() {
244					Some('=') => punct!(2, SlashEq),
245					_ => punct!(1, Slash),
246				},
247				'^' => match s.pop_first() {
248					Some('=') => punct!(2, CaretEq),
249					_ => punct!(1, Caret),
250				},
251				'%' => match s.pop_first() {
252					Some('=') => punct!(2, PercentEq),
253					_ => punct!(1, Percent),
254				},
255				'.' => match s.pop_first() {
256					Some('.') => match s.pop_first() {
257						Some('.') => punct!(3, DotDotDot),
258						Some('=') => punct!(3, DotDotEq),
259						_ => punct!(2, DotDot),
260					},
261					_ => punct!(1, Dot),
262				},
263				':' => match s.pop_first() {
264					Some(':') => punct!(2, PathSep),
265					_ => punct!(1, Colon),
266				},
267				'~' => punct!(1, Tilde),
268				'@' => punct!(1, At),
269				',' => punct!(1, Comma),
270				';' => punct!(1, Semi),
271				'#' => punct!(1, Pound),
272				'$' if !skip_dollar => punct!(1, Dollar),
273				'?' if !skip_question => punct!(1, Question),
274				'_' => punct!(1, Underscore),
275				'(' if !skip_delimiters => punct!(1, ParenOpen),
276				')' if !skip_delimiters => punct!(1, ParenClose),
277				'[' if !skip_delimiters => punct!(1, BracketOpen),
278				']' if !skip_delimiters => punct!(1, BracketClose),
279				'{' if !skip_delimiters => punct!(1, BracesOpen),
280				'}' if !skip_delimiters => punct!(1, BracesClose),
281				_ => None,
282			}
283		});
284		let (s, punct) = res.ok_or(PunctuationError::NotFound)?;
285
286		let punct = match punct {
287			Punct::Eq => Self::Eq(Eq(ws, s)),
288			Punct::Lt => Self::Lt(Lt(ws, s)),
289			Punct::Le => Self::Le(Le(ws, s)),
290			Punct::EqEq => Self::EqEq(EqEq(ws, s)),
291			Punct::Ne => Self::Ne(Ne(ws, s)),
292			Punct::Ge => Self::Ge(Ge(ws, s)),
293			Punct::Gt => Self::Gt(Gt(ws, s)),
294			Punct::AndAnd => Self::AndAnd(AndAnd(ws, s)),
295			Punct::OrOr => Self::OrOr(OrOr(ws, s)),
296			Punct::Not => Self::Not(Not(ws, s)),
297			Punct::Tilde => Self::Tilde(Tilde(ws, s)),
298			Punct::Plus => Self::Plus(Plus(ws, s)),
299			Punct::Minus => Self::Minus(Minus(ws, s)),
300			Punct::Star => Self::Star(Star(ws, s)),
301			Punct::Slash => Self::Slash(Slash(ws, s)),
302			Punct::Percent => Self::Percent(Percent(ws, s)),
303			Punct::Caret => Self::Caret(Caret(ws, s)),
304			Punct::And => Self::And(And(ws, s)),
305			Punct::Or => Self::Or(Or(ws, s)),
306			Punct::Shl => Self::Shl(Shl(ws, s)),
307			Punct::Shr => Self::Shr(Shr(ws, s)),
308			Punct::PlusEq => Self::PlusEq(PlusEq(ws, s)),
309			Punct::MinusEq => Self::MinusEq(MinusEq(ws, s)),
310			Punct::StarEq => Self::StarEq(StarEq(ws, s)),
311			Punct::SlashEq => Self::SlashEq(SlashEq(ws, s)),
312			Punct::PercentEq => Self::PercentEq(PercentEq(ws, s)),
313			Punct::CaretEq => Self::CaretEq(CaretEq(ws, s)),
314			Punct::AndEq => Self::AndEq(AndEq(ws, s)),
315			Punct::OrEq => Self::OrEq(OrEq(ws, s)),
316			Punct::ShlEq => Self::ShlEq(ShlEq(ws, s)),
317			Punct::ShrEq => Self::ShrEq(ShrEq(ws, s)),
318			Punct::At => Self::At(At(ws, s)),
319			Punct::Dot => Self::Dot(Dot(ws, s)),
320			Punct::DotDot => Self::DotDot(DotDot(ws, s)),
321			Punct::DotDotDot => Self::DotDotDot(DotDotDot(ws, s)),
322			Punct::DotDotEq => Self::DotDotEq(DotDotEq(ws, s)),
323			Punct::Comma => Self::Comma(Comma(ws, s)),
324			Punct::Semi => Self::Semi(Semi(ws, s)),
325			Punct::Colon => Self::Colon(Colon(ws, s)),
326			Punct::PathSep => Self::PathSep(PathSep(ws, s)),
327			Punct::RArrow => Self::RArrow(RArrow(ws, s)),
328			Punct::LArrow => Self::LArrow(LArrow(ws, s)),
329			Punct::FatArrow => Self::FatArrow(FatArrow(ws, s)),
330			Punct::Pound => Self::Pound(Pound(ws, s)),
331			Punct::Dollar => Self::Dollar(Dollar(ws, s)),
332			Punct::Question => Self::Question(Question(ws, s)),
333			Punct::Underscore => Self::Underscore(Underscore(ws, s)),
334			Punct::ParenOpen => Self::ParenOpen(ParenOpen(ws, s)),
335			Punct::ParenClose => Self::ParenClose(ParenClose(ws, s)),
336			Punct::BracketOpen => Self::BracketOpen(BracketOpen(ws, s)),
337			Punct::BracketClose => Self::BracketClose(BracketClose(ws, s)),
338			Punct::BracesOpen => Self::BracesOpen(BracesOpen(ws, s)),
339			Punct::BracesClose => Self::BracesClose(BracesClose(ws, s)),
340		};
341
342		Ok(punct)
343	}
344}
345
346#[derive(derive_more::Debug, derive_more::From, ParseError)]
347pub enum PunctuationError {
348	#[parse_error(transparent)]
349	Whitespace(ParserError<Whitespace>),
350
351	#[parse_error(fmt = "Expected punctuation")]
352	NotFound,
353}
rustidy_ast/token.rs

rustidy_ast/
token.rs
