lemmy_help/
lexer.rs

1mod token;
2pub use token::*;
3
4use std::ops::Range;
5
6use chumsky::{
7    prelude::{any, choice, end, filter, just, take_until, Simple},
8    recursive::recursive,
9    text::{ident, keyword, newline, whitespace, TextParser},
10    Parser,
11};
12
13type Spanned = (TagType, Range<usize>);
14
15const C: [char; 3] = ['.', '_', '-'];
16
17#[derive(Debug)]
18pub struct Lexer;
19
20impl Lexer {
21    /// Parse emmylua/lua files into rust token
22    pub fn init() -> impl Parser<char, Vec<Spanned>, Error = Simple<char>> {
23        let triple = just("---");
24        let space = just(' ').repeated().at_least(1);
25        let till_eol = take_until(newline());
26
27        let comment = till_eol.map(|(x, _)| x.iter().collect());
28        let desc = space.ignore_then(comment).or_not();
29
30        let public = keyword("public").to(Scope::Public);
31        let private = keyword("private")
32            .to(Scope::Private)
33            .or(keyword("protected").to(Scope::Protected))
34            .or(keyword("package").to(Scope::Package));
35
36        let hidden = private
37            .clone()
38            .ignore_then(newline())
39            .then_ignore(choice((
40                // eat up all the emmylua, if any, then one valid token
41                triple
42                    .then(till_eol)
43                    .padded()
44                    .repeated()
45                    .ignore_then(ident()),
46                // if there is no emmylua, just eat the next token
47                // so the next parser won't recognize the code
48                ident().padded(),
49            )))
50            .ignored();
51
52        let union_literal = choice((
53            just('\'')
54                .ignore_then(filter(|c| c != &'\'').repeated())
55                .then_ignore(just('\''))
56                .collect()
57                .map(Member::Literal),
58            just('`')
59                .ignore_then(filter(|c| c != &'`').repeated())
60                .then_ignore(just('`'))
61                .collect()
62                .map(Member::Ident),
63        ));
64
65        let variant = just('|')
66            .then_ignore(space)
67            .ignore_then(union_literal)
68            .then(
69                space
70                    .ignore_then(just('#').ignore_then(space).ignore_then(comment))
71                    .or_not(),
72            )
73            .map(|(t, d)| TagType::Variant(t, d));
74
75        let optional = just('?').or_not().map(|c| match c {
76            Some(_) => Name::Opt as fn(_) -> _,
77            None => Name::Req as fn(_) -> _,
78        });
79
80        let name = filter(|x: &char| x.is_alphanumeric() || C.contains(x))
81            .repeated()
82            .collect();
83
84        let ty = recursive(|inner| {
85            let comma = just(',').padded();
86            let colon = just(':').padded();
87
88            let any = just("any").to(Ty::Any);
89            let unknown = just("unknown").to(Ty::Unknown);
90            let nil = just("nil").to(Ty::Nil);
91            let boolean = just("boolean").to(Ty::Boolean);
92            let string = just("string").to(Ty::String);
93            let num = just("number").to(Ty::Number);
94            let int = just("integer").to(Ty::Integer);
95            let function = just("function").to(Ty::Function);
96            let thread = just("thread").to(Ty::Thread);
97            let userdata = just("userdata").to(Ty::Userdata);
98            let lightuserdata = just("lightuserdata").to(Ty::Lightuserdata);
99
100            #[inline]
101            fn array_union(
102                p: impl Parser<char, Ty, Error = Simple<char>>,
103                inner: impl Parser<char, Ty, Error = Simple<char>>,
104            ) -> impl Parser<char, Ty, Error = Simple<char>> {
105                p.then(just("[]").repeated())
106                    .foldl(|arr, _| Ty::Array(Box::new(arr)))
107                    // NOTE: Not the way I wanted i.e., Ty::Union(Vec<Ty>) it to be, but it's better than nothing
108                    .then(just('|').padded().ignore_then(inner).repeated())
109                    .foldl(|x, y| Ty::Union(Box::new(x), Box::new(y)))
110            }
111
112            let list_like = ident()
113                .padded()
114                .then(optional)
115                .then(
116                    colon
117                        .ignore_then(inner.clone())
118                        .or_not()
119                        // NOTE: if param type is missing then LLS treats it as `any`
120                        .map(|x| x.unwrap_or(Ty::Any)),
121                )
122                .map(|((n, attr), t)| (attr(n), t))
123                .separated_by(comma)
124                .allow_trailing();
125
126            let fun = just("fun")
127                .ignore_then(
128                    list_like
129                        .clone()
130                        .delimited_by(just('(').then(whitespace()), whitespace().then(just(')'))),
131                )
132                .then(
133                    colon
134                        .ignore_then(inner.clone().separated_by(comma))
135                        .or_not(),
136                )
137                .map(|(param, ret)| Ty::Fun(param, ret));
138
139            let table = just("table")
140                .ignore_then(
141                    just('<')
142                        .ignore_then(inner.clone().map(Box::new))
143                        .then_ignore(comma)
144                        .then(inner.clone().map(Box::new))
145                        .then_ignore(just('>'))
146                        .or_not(),
147                )
148                .map(Ty::Table);
149
150            let dict = list_like
151                .delimited_by(just('{').then(whitespace()), whitespace().then(just('}')))
152                .map(Ty::Dict);
153
154            let ty_name = name.map(Ty::Ref);
155
156            let parens = inner
157                .clone()
158                .delimited_by(just('(').padded(), just(')').padded());
159
160            // Union of string literals: '"g@"'|'"g@$"'
161            let string_literal = union_literal.map(Ty::Member);
162
163            choice((
164                array_union(any, inner.clone()),
165                array_union(unknown, inner.clone()),
166                array_union(nil, inner.clone()),
167                array_union(boolean, inner.clone()),
168                array_union(string, inner.clone()),
169                array_union(num, inner.clone()),
170                array_union(int, inner.clone()),
171                array_union(function, inner.clone()),
172                array_union(thread, inner.clone()),
173                array_union(userdata, inner.clone()),
174                array_union(lightuserdata, inner.clone()),
175                array_union(fun, inner.clone()),
176                array_union(table, inner.clone()),
177                array_union(dict, inner.clone()),
178                array_union(parens, inner.clone()),
179                array_union(string_literal, inner.clone()),
180                array_union(ty_name, inner),
181            ))
182        });
183
184        let code_lang = ident().then_ignore(space).or_not();
185
186        let tag = just('@').ignore_then(choice((
187            hidden.or(public.clone().ignored()).to(TagType::Skip),
188            just("toc")
189                .ignore_then(space)
190                .ignore_then(comment)
191                .map(TagType::Toc),
192            just("mod")
193                .then_ignore(space)
194                .ignore_then(name)
195                .then(desc)
196                .map(|(name, desc)| TagType::Module(name, desc)),
197            just("divider")
198                .ignore_then(space)
199                .ignore_then(any())
200                .map(TagType::Divider),
201            just("brief").ignore_then(space).ignore_then(choice((
202                just("[[").to(TagType::BriefStart),
203                just("]]").to(TagType::BriefEnd),
204            ))),
205            just("param")
206                .ignore_then(space)
207                .ignore_then(choice((
208                    just("...").map(|n| Name::Req(n.to_string())),
209                    ident().then(optional).map(|(n, o)| o(n)),
210                )))
211                .then_ignore(space)
212                .then(ty.clone())
213                .then(desc)
214                .map(|((name, ty), desc)| TagType::Param(name, ty, desc)),
215            just("return")
216                .ignore_then(space)
217                .ignore_then(ty.clone())
218                .then(choice((
219                    newline().to((None, None)),
220                    space.ignore_then(choice((
221                        just('#').ignore_then(comment).map(|x| (None, Some(x))),
222                        ident().then(desc).map(|(name, desc)| (Some(name), desc)),
223                    ))),
224                )))
225                .map(|(ty, (name, desc))| TagType::Return(ty, name, desc)),
226            just("class")
227                .ignore_then(space)
228                .ignore_then(name)
229                .then(just(':').padded().ignore_then(ident()).or_not())
230                .map(|(name, parent)| TagType::Class(name, parent)),
231            just("field")
232                .ignore_then(space.ignore_then(private.or(public)).or_not())
233                .then_ignore(space)
234                .then(ident())
235                .then(optional)
236                .then_ignore(space)
237                .then(ty.clone())
238                .then(desc)
239                .map(|((((scope, name), opt), ty), desc)| {
240                    TagType::Field(scope.unwrap_or(Scope::Public), opt(name), ty, desc)
241                }),
242            just("alias")
243                .ignore_then(space)
244                .ignore_then(name)
245                .then(space.ignore_then(ty.clone()).or_not())
246                .map(|(name, ty)| TagType::Alias(name, ty)),
247            just("type")
248                .ignore_then(space)
249                .ignore_then(ty)
250                .then(desc)
251                .map(|(ty, desc)| TagType::Type(ty, desc)),
252            just("tag")
253                .ignore_then(space)
254                .ignore_then(comment)
255                .map(TagType::Tag),
256            just("see")
257                .ignore_then(space)
258                .ignore_then(comment)
259                .map(TagType::See),
260            just("usage").ignore_then(space).ignore_then(choice((
261                code_lang
262                    .then(
263                        just('`')
264                            .ignore_then(filter(|c| *c != '`').repeated())
265                            .then_ignore(just('`'))
266                            .collect(),
267                    )
268                    .map(|(lang, code)| TagType::Usage(lang, code)),
269                code_lang.then_ignore(just("[[")).map(TagType::UsageStart),
270                just("]]").to(TagType::UsageEnd),
271            ))),
272            just("export")
273                .ignore_then(space)
274                .ignore_then(ident())
275                .then_ignore(take_until(end()))
276                .map(TagType::Export),
277        )));
278
279        let func = keyword("function").padded();
280        let ret = keyword("return");
281        let assign = just('=').padded();
282
283        // obj = ID (prop)+ "="
284        // fn = ID (prop | colon_op)
285        // prop = (dot_op)+ ("(" | colon_op)
286        // dot_op = "." ID
287        // colon_op = ":" ID "("
288        let colon_op = just(':')
289            .ignore_then(ident())
290            .then_ignore(just('('))
291            .map(Op::Colon);
292
293        let dot_op = just('.')
294            .ignore_then(ident().map(Op::Dot))
295            .repeated()
296            .at_least(1);
297
298        let prop = dot_op
299            .then(choice((just('(').to(None), colon_op.map(Some))))
300            .map(|(mut props, meth)| {
301                if let Some(x) = meth {
302                    props.push(x)
303                }
304                Op::Deep(props)
305            });
306
307        let dotted = ident()
308            .then(choice((prop, colon_op)))
309            .map(|(prefix, op)| (prefix, op));
310
311        let expr = ident().then(dot_op).then_ignore(assign);
312
313        choice((
314            triple.ignore_then(choice((tag, variant, comment.map(TagType::Comment)))),
315            func.clone()
316                .ignore_then(dotted)
317                .map(|(prefix, op)| TagType::Func(prefix, op)),
318            expr.then(func.or_not())
319                .map(|((prefix, op), is_fn)| match is_fn {
320                    Some(_) => TagType::Func(prefix, Op::Deep(op)),
321                    None => TagType::Expr(prefix, Op::Deep(op)),
322                }),
323            ret.ignore_then(ident().padded())
324                .then_ignore(end())
325                .map(TagType::Export),
326            till_eol.to(TagType::Skip),
327        ))
328        .padded()
329        .map_with_span(|t, r| (t, r))
330        .repeated()
331    }
332}
lemmy_help/lexer.rs

lemmy_help/
lexer.rs
