eson_tokenizer/
token.rs

1use std::fmt::{Display, Formatter};
2
3use nom::Slice;
4
5use crate::Span;
6
7#[derive(Debug, PartialEq, Clone, Eq)]
8pub struct TokenPosition {
9    pub line: u32,
10    pub column: usize,
11}
12
13impl<'a> From<Span<'a>> for TokenPosition {
14    fn from(span: Span) -> Self {
15        TokenPosition {
16            line: span.location_line(),
17            column: span.get_utf8_column(),
18        }
19    }
20}
21
22#[derive(Debug, PartialEq, Clone, Eq)]
23pub struct TokenRange(pub TokenPosition, pub TokenPosition);
24
25impl Default for TokenRange {
26    fn default() -> Self {
27        TokenRange(
28            TokenPosition { line: 0, column: 0 },
29            TokenPosition { line: 0, column: 0 },
30        )
31    }
32}
33
34impl From<(Span<'_>, Span<'_>)> for TokenRange {
35    fn from((start, end): (Span, Span)) -> TokenRange {
36        TokenRange(TokenPosition::from(start), TokenPosition::from(end))
37    }
38}
39
40impl From<(Span<'_>, usize)> for TokenRange {
41    fn from((start, len): (Span, usize)) -> TokenRange {
42        TokenRange(
43            TokenPosition::from(start),
44            TokenPosition::from(start.slice(len..)),
45        )
46    }
47}
48
49impl From<(Span<'_>, usize, usize)> for TokenRange {
50    fn from((s, abs_offset_i, abs_offset_j): (Span, usize, usize)) -> Self {
51        TokenRange(
52            TokenPosition::from(s.slice(abs_offset_i..)),
53            TokenPosition::from(s.slice(abs_offset_j..)),
54        )
55    }
56}
57
58#[derive(Debug, PartialEq, Clone, Eq)]
59pub enum TokenKey {
60    Dummy,
61    DummySn(usize),
62    Sn(usize, TokenRange),
63    String(String, TokenRange),
64}
65
66#[derive(Debug, PartialEq, Clone)]
67pub struct TokenId(pub String, pub TokenRange);
68
69impl Display for TokenId {
70    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
71        write!(f, "{}", self.0)
72    }
73}
74
75impl TokenId {
76    pub fn name(&self) -> &str {
77        &self.0.as_str()
78    }
79}
80
81// TokenDecorator
82#[derive(Debug, PartialEq, Clone)]
83pub struct TokenDec(pub TokenId, pub Vec<Token>, pub TokenRange);
84
85#[derive(Debug, PartialEq, Clone)]
86pub enum Token {
87    Void,
88    TokenPrimNull(TokenRange),
89    TokenPrimNumberInt(i64, TokenRange),
90    TokenPrimNumberFloat(f64, TokenRange),
91    TokenPrimString(String, TokenRange),
92    TokenPrimBoolean(bool, TokenRange),
93    TokenVar(Vec<TokenKey>, TokenRange),
94    TokenComment(String, TokenRange),
95    TokenRefVarSibling(Vec<TokenKey>, TokenRange),
96    TokenRefVarUncle(Vec<TokenKey>, TokenRange),
97    TokenRefVarRoot(Vec<TokenKey>, TokenRange),
98    TokenFnCall(TokenId, Vec<Token>, TokenRange),
99    TokenExprSequence(Vec<Token>, TokenRange),
100    TokenFrameExprGroup(Box<Token>, TokenRange),
101    TokenFrameRoot(Vec<TokenDec>, Box<Token>, TokenRange), // (decorators, body)
102    TokenFrameList(Vec<(TokenKey, Vec<TokenDec>, Token)>, TokenRange), // Vec<(index, decorators, body)>
103    TokenFrameDict(Vec<(TokenKey, Vec<TokenDec>, Token)>, TokenRange), // Vec<(index, decorators, body)>
104    TokenOpPipe(TokenRange),
105    TokenOpEq(TokenRange),
106    TokenOpNe(TokenRange),
107    TokenOpLe(TokenRange),
108    TokenOpGe(TokenRange),
109    TokenOpAnd(TokenRange),
110    TokenOpOr(TokenRange),
111    TokenOpNot(TokenRange),
112    TokenOpGt(TokenRange),
113    TokenOpLt(TokenRange),
114    TokenOpConcat(TokenRange),
115    TokenDummyOpConcat,
116    TokenOpAdd(TokenRange),
117    TokenOpSub(TokenRange),
118    TokenOpMul(TokenRange),
119    TokenOpDiv(TokenRange),
120    TokenOpMod(TokenRange),
121    TokenOpQ(TokenRange),
122    TokenOpColon(TokenRange),
123}
124
125impl Display for TokenKey {
126    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
127        match self {
128            TokenKey::Dummy => write!(f, "_"),
129            TokenKey::DummySn(i) => write!(f, "{}", i),
130            TokenKey::Sn(i, ..) => write!(f, "{}", i),
131            TokenKey::String(s, ..) => write!(f, "{}", s),
132        }
133    }
134}
135
136impl Default for Token {
137    fn default() -> Self {
138        Token::Void
139    }
140}
141
142impl Display for Token {
143    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
144        match self {
145            Token::Void => write!(f, "()"),
146            Token::TokenPrimNumberInt(v, _) => write!(f, "{}", v),
147            Token::TokenPrimNumberFloat(v, _) => write!(f, "{}", v),
148            Token::TokenPrimString(v, _) => write!(f, "{}", v),
149            Token::TokenPrimBoolean(v, _) => write!(f, "{}", v),
150            Token::TokenPrimNull(_) => write!(f, "null"),
151            Token::TokenFnCall(n, _, _) => write!(f, "{}(..)", n),
152            Token::TokenVar(v, _) => write!(
153                f,
154                "${}",
155                v.iter()
156                    .map(|v| v.to_string())
157                    .collect::<Vec<String>>()
158                    .join(".")
159            ),
160            Token::TokenComment(v, _) => write!(f, "/* {} */", v),
161            Token::TokenRefVarSibling(v, _) => write!(f, "&sibling.{:?}", v),
162            Token::TokenRefVarUncle(v, _) => write!(f, "&uncle.{:?}", v),
163            Token::TokenRefVarRoot(v, _) => write!(f, "&root.{:?}", v),
164            Token::TokenFrameRoot(_, _, _) => write!(f, "base{{..}}"),
165            Token::TokenFrameDict(_, _) => write!(f, "{{..}}"),
166            Token::TokenFrameList(_, _) => write!(f, "[..]"),
167            Token::TokenExprSequence(s, _) => write!(f, "{:?}", s),
168            Token::TokenFrameExprGroup(s, _) => write!(f, "{:?}", s),
169            Token::TokenOpPipe(_) => write!(f, "|"),
170            Token::TokenOpEq(_) => write!(f, "=="),
171            Token::TokenOpNe(_) => write!(f, "!="),
172            Token::TokenOpLe(_) => write!(f, "<="),
173            Token::TokenOpGe(_) => write!(f, ">="),
174            Token::TokenOpAnd(_) => write!(f, "&&"),
175            Token::TokenOpOr(_) => write!(f, "||"),
176            Token::TokenOpNot(_) => write!(f, "!"),
177            Token::TokenOpGt(_) => write!(f, ">"),
178            Token::TokenOpLt(_) => write!(f, "<"),
179            Token::TokenOpConcat(_) => write!(f, ".."),
180            Token::TokenDummyOpConcat => write!(f, ".."),
181            Token::TokenOpAdd(_) => write!(f, "+"),
182            Token::TokenOpSub(_) => write!(f, "-"),
183            Token::TokenOpMul(_) => write!(f, "*"),
184            Token::TokenOpDiv(_) => write!(f, "/"),
185            Token::TokenOpMod(_) => write!(f, "%"),
186            Token::TokenOpQ(_) => write!(f, "?"),
187            Token::TokenOpColon(_) => write!(f, ":"),
188        }
189    }
190}
191
192pub trait ReadTokenRange {
193    fn range(&self) -> TokenRange;
194}
195
196impl ReadTokenRange for Token {
197    fn range(&self) -> TokenRange {
198        match self {
199            Token::Void => TokenRange::default(),
200            Token::TokenPrimNumberInt(_, r) => r.clone(),
201            Token::TokenPrimNumberFloat(_, r) => r.clone(),
202            Token::TokenPrimString(_, r) => r.clone(),
203            Token::TokenPrimBoolean(_, r) => r.clone(),
204            Token::TokenPrimNull(r) => r.clone(),
205            Token::TokenFnCall(_, _, r) => r.clone(),
206            Token::TokenVar(_, r) => r.clone(),
207            Token::TokenComment(_, r) => r.clone(),
208            Token::TokenRefVarSibling(_, r) => r.clone(),
209            Token::TokenRefVarUncle(_, r) => r.clone(),
210            Token::TokenRefVarRoot(_, r) => r.clone(),
211            Token::TokenFrameRoot(_, _, r) => r.clone(),
212            Token::TokenFrameDict(_, r) => r.clone(),
213            Token::TokenFrameList(_, r) => r.clone(),
214            Token::TokenExprSequence(_, r) => r.clone(),
215            Token::TokenFrameExprGroup(_, r) => r.clone(),
216            Token::TokenOpPipe(r) => r.clone(),
217            Token::TokenOpEq(r) => r.clone(),
218            Token::TokenOpNe(r) => r.clone(),
219            Token::TokenOpLe(r) => r.clone(),
220            Token::TokenOpGe(r) => r.clone(),
221            Token::TokenOpAnd(r) => r.clone(),
222            Token::TokenOpOr(r) => r.clone(),
223            Token::TokenOpNot(r) => r.clone(),
224            Token::TokenOpGt(r) => r.clone(),
225            Token::TokenOpLt(r) => r.clone(),
226            Token::TokenOpConcat(r) => r.clone(),
227            Token::TokenDummyOpConcat => TokenRange::default(),
228            Token::TokenOpAdd(r) => r.clone(),
229            Token::TokenOpSub(r) => r.clone(),
230            Token::TokenOpMul(r) => r.clone(),
231            Token::TokenOpDiv(r) => r.clone(),
232            Token::TokenOpMod(r) => r.clone(),
233            Token::TokenOpQ(r) => r.clone(),
234            Token::TokenOpColon(r) => r.clone(),
235        }
236    }
237}
238
239impl ReadTokenRange for TokenDec {
240    fn range(&self) -> TokenRange {
241        self.2.clone()
242    }
243}
244
245impl ReadTokenRange for TokenKey {
246    fn range(&self) -> TokenRange {
247        match self {
248            TokenKey::Dummy => TokenRange::default(),
249            TokenKey::DummySn(_) => TokenRange::default(),
250            TokenKey::Sn(_, r) => r.clone(),
251            TokenKey::String(_, r) => r.clone(),
252        }
253    }
254}
255
256impl ReadTokenRange for TokenId {
257    fn range(&self) -> TokenRange {
258        self.1.clone()
259    }
260}
261
262#[cfg(test)]
263mod tests {
264    use crate::TokenDec;
265
266    #[test]
267    fn test_read_token_range() {
268        use crate::token::TokenPosition;
269        use crate::token::{ReadTokenRange, Token, TokenId, TokenKey, TokenRange};
270        fn random_token_range() -> TokenRange {
271            let i = rand::random::<u8>();
272            let m = rand::random::<u16>();
273            let j = rand::random::<u8>();
274            let n = rand::random::<u16>();
275
276            TokenRange(
277                TokenPosition {
278                    line: i as u32,
279                    column: m as usize,
280                },
281                TokenPosition {
282                    line: (i as u32) + (j as u32),
283                    column: n as usize,
284                },
285            )
286        }
287
288        let r = random_token_range();
289        let t = Token::TokenPrimNumberInt(1, r.clone());
290        assert_eq!(t.range(), r);
291
292        let r = random_token_range();
293        let t = Token::TokenPrimString("hello".to_string(), r.clone());
294        assert_eq!(t.range(), r);
295
296        let r = random_token_range();
297        let t = Token::TokenRefVarSibling(vec![TokenKey::Dummy], r.clone());
298        assert_eq!(t.range(), r);
299
300        let r = random_token_range();
301        let k = TokenKey::String("hello".to_string(), r.clone());
302        assert_eq!(k.range(), r);
303
304        let k = TokenKey::Dummy;
305        assert_eq!(k.range(), TokenRange::default());
306
307        let r = random_token_range();
308        let i = TokenId("hello".to_string(), r.clone());
309        assert_eq!(i.range(), r);
310
311        let r = random_token_range();
312        let d = TokenDec(i, vec![], r.clone());
313        assert_eq!(d.range(), r);
314    }
315}
eson_tokenizer/token.rs

eson_tokenizer/
token.rs
