mazer_core/
tokenizer.rs

1use rayon::prelude::*;
2use unicode_segmentation::UnicodeSegmentation;
3
4use crate::pretty_err::{DebugContext, ErrorKind};
5
6#[derive(Debug, Clone)]
7pub enum MarkdownTag {
8    Header(HeaderKind, String),
9    LineSeparator,
10    Checkbox(bool, String),
11    BulletPoint(String),
12    Blockquote(String),
13    CodeBlock(String),
14    Link(LinkKind, String, String),
15}
16
17#[derive(Debug, Clone)]
18pub enum LinkKind {
19    Image,
20    Hyperlink,
21}
22
23#[derive(Debug, Clone)]
24pub enum HeaderKind {
25    H1,
26    H2,
27    H3,
28}
29
30impl From<HeaderKind> for usize {
31    fn from(value: HeaderKind) -> Self {
32        match value {
33            HeaderKind::H1 => 1,
34            HeaderKind::H2 => 2,
35            HeaderKind::H3 => 3,
36        }
37    }
38}
39
40impl From<usize> for HeaderKind {
41    fn from(val: usize) -> Self {
42        match val {
43            1 => HeaderKind::H1,
44            2 => HeaderKind::H2,
45            3 => HeaderKind::H3,
46            _ => HeaderKind::H1,
47        }
48    }
49}
50
51#[derive(Debug, Clone)]
52pub enum Token {
53    LetExpr(String, String),
54    Fn(FnKind, String),
55    Literal(String),
56    Text(Option<Emphasis>, String),
57    Comment(String),
58    Markdown(MarkdownTag),
59    Newline,
60}
61
62#[derive(Debug, Clone)]
63pub enum FnKind {
64    Fmt,
65    Eval,
66}
67
68impl From<FnKind> for String {
69    fn from(value: FnKind) -> Self {
70        match value {
71            FnKind::Fmt => "fmt".to_string(),
72            FnKind::Eval => "eval".to_string(),
73        }
74    }
75}
76
77#[derive(Debug, Clone, PartialEq)]
78pub enum Emphasis {
79    Bold,
80    Italic,
81    Strikethrough,
82}
83
84#[derive(Debug)]
85pub struct Lexer {
86    src: Vec<String>,
87    pos: usize,
88    line: usize,
89    max: usize,
90    prv: Option<String>,
91    ctx: DebugContext,
92}
93
94impl Lexer {
95    pub fn new(src: String, ctx: DebugContext) -> Self {
96        let uni_vec = UnicodeSegmentation::graphemes(src.as_str(), true)
97            .collect::<Vec<&str>>()
98            .par_iter()
99            .map(|&x| x.to_string())
100            .collect::<Vec<String>>();
101
102        // push 5 newlines to the end of the source code to ensure
103        // that the last line is parsed and it does not run out of
104        // bounds
105        let mut uni_vec = uni_vec;
106        for _ in 0..5 {
107            uni_vec.push("\n".to_string());
108        }
109
110        let max = uni_vec.len();
111        Lexer {
112            src: uni_vec,
113            pos: 0,
114            line: 0,
115            max,
116            prv: None,
117            ctx,
118        }
119    }
120
121    fn create_error(&mut self, err: ErrorKind) {
122        let src = self.src.join("");
123        let src = src.split("\n").collect::<Vec<&str>>();
124
125        let err_line = src[self.line - 1].to_string();
126        self.ctx.set_source_code(err_line);
127        self.ctx.set_position(self.pos);
128        self.ctx.set_error(err);
129    }
130
131    fn char(&mut self) -> Result<String, DebugContext> {
132        if self.pos >= self.max {
133            // [ERROR]
134            let e = ErrorKind::AbruptAdieu(format!(
135                "Reached the end of file looking for position {}",
136                self.pos
137            ));
138            self.create_error(e);
139            return Err(self.ctx.clone());
140        }
141        Ok(self.src[self.pos].clone())
142    }
143
144    fn peek(&mut self) -> Result<String, DebugContext> {
145        if self.pos >= self.max {
146            // [ERROR]
147            let e = ErrorKind::AbruptAdieu(format!(
148                "Reached the end of file looking for position {}",
149                self.pos
150            ));
151            self.create_error(e);
152            Err(self.ctx.clone())
153        } else {
154            Ok(self.src[self.pos + 1].clone())
155        }
156    }
157
158    // peeks the char after the next char
159    fn peek_n(&mut self, n: usize) -> Result<String, DebugContext> {
160        if self.pos >= self.max {
161            // [ERROR]
162            let e = ErrorKind::AbruptAdieu(format!(
163                "Reached the end of file looking for position {}",
164                self.pos
165            ));
166            self.create_error(e);
167            Err(self.ctx.clone())
168        } else {
169            Ok(self.src[self.pos + n].clone())
170        }
171    }
172
173    fn advance_char(&mut self) -> Result<(), DebugContext> {
174        if self.pos >= self.max {
175            let e = ErrorKind::AbruptAdieu(format!(
176                "Reached the end of file looking for position {}",
177                self.pos
178            ));
179            self.create_error(e);
180            return Err(self.ctx.clone());
181        }
182        self.pos += 1;
183        Ok(())
184    }
185
186    fn must_consume(&mut self, c: &str) -> Result<(), DebugContext> {
187        let curr = self.char()?;
188        // [ERROR]
189        if curr != c {
190            let e = ErrorKind::BrokenExpectations(format!("Expected '{}' but found '{}'", c, curr));
191            self.create_error(e);
192            return Err(self.ctx.clone());
193        }
194        self.advance_char()?;
195        Ok(())
196    }
197
198    fn consume_whitespace(&mut self) -> Result<(), DebugContext> {
199        // keep moving forward if current string is made up of
200        // whitespaces
201        while self.char()?.trim().is_empty() {
202            self.advance_char()?;
203        }
204
205        Ok(())
206    }
207
208    fn consume_until_not(&mut self, c: &str) -> Result<String, DebugContext> {
209        let start = self.pos;
210        while self.char()? == c {
211            self.pos += 1;
212        }
213
214        Ok(self.src[start..self.pos].join(""))
215    }
216
217    fn consume_till(&mut self, c: &str) -> Result<String, DebugContext> {
218        let start = self.pos;
219        while self.char()? != c {
220            self.pos += 1;
221        }
222        Ok(self.src[start..self.pos].join(""))
223    }
224
225    fn consume_line(&mut self) -> Result<String, DebugContext> {
226        self.consume_till("\n")
227    }
228
229    fn consume_nested_parenthesis(&mut self) -> Result<String, DebugContext> {
230        // iterate over source from current position
231        // keep adding when ( is encountered
232        // and decreasing when ) is encountered
233        // if underflow then less opening
234        // if overflow or reaches end of file then
235        let mut store = String::from(self.char()?);
236        let mut count = 1;
237
238        while count > 0 {
239            self.advance_char()?;
240            if self.pos >= self.max {
241                // [ERROR]
242                let e = ErrorKind::LonelyParenthesis("Unmatched parenthesis".to_string());
243                self.create_error(e);
244                return Err(self.ctx.clone());
245            }
246
247            let curr = self.char()?;
248            store.push_str(&curr);
249
250            if curr == "(" {
251                count += 1;
252            } else if curr == ")" {
253                count -= 1;
254            }
255        }
256
257        // check if balanced
258        if count != 0 {
259            // [ERROR]
260            let e = ErrorKind::LonelyParenthesis("Unmatched parenthesis".to_string());
261            self.create_error(e);
262            return Err(self.ctx.clone());
263        }
264
265        Ok(store)
266    }
267
268    pub fn next_line(&mut self) -> Result<Option<Vec<Token>>, DebugContext> {
269        self.line += 1;
270        if self.pos >= self.max {
271            return Ok(None);
272        }
273
274        let mut tokens: Vec<Token> = Vec::new();
275        while let Some(tok) = self.next_token()? {
276            tokens.push(tok);
277        }
278        self.advance_char()?;
279
280        if tokens.is_empty() {
281            return Ok(Some(vec![Token::Newline]));
282        }
283
284        Ok(Some(tokens))
285    }
286
287    fn next_token(&mut self) -> Result<Option<Token>, DebugContext> {
288        if self.pos >= self.max || self.char()? == "\n" {
289            if self.char()? == "\n" {
290                self.prv = Some("\n".to_string());
291            }
292            return Ok(None);
293        }
294
295        let curr_tok = self.char()?;
296        self.prv = None;
297
298        // consume comments
299        if curr_tok == "/" && self.peek()? == "/" {
300            self.advance_char()?;
301            self.advance_char()?;
302            let comment = self.consume_line()?;
303            let comment = comment.trim();
304            Ok(Some(Token::Comment(comment.to_string())))
305        // literals
306        } else if curr_tok == "\"" {
307            self.advance_char()?;
308            let literal = self.consume_till("\"")?.to_string();
309            self.must_consume("\"")?;
310
311            return Ok(Some(Token::Literal(literal)));
312        // let statements
313        } else if curr_tok == "l" && self.peek()? == "e" && self.peek_n(2)? == "t" {
314            self.advance_char()?;
315            self.advance_char()?;
316            self.advance_char()?;
317
318            if self.char()? != " " {
319                // [ERROR]
320                let e = ErrorKind::GrammarGoblin(
321                    "Let statement should be followed by a space".to_string(),
322                );
323                self.create_error(e);
324                return Err(self.ctx.clone());
325            }
326
327            let var = self.consume_till("=")?.trim().to_string();
328            // [ERROR]
329            if var.is_empty() {
330                let e = ErrorKind::NamelessNomad("Variable name cannot be empty".to_string());
331                self.create_error(e);
332                return Err(self.ctx.clone());
333            }
334
335            self.must_consume("=")?;
336            let mut val = self.consume_till(";")?.trim().to_string();
337            self.must_consume(";")?;
338
339            val.push_str(";");
340
341            return Ok(Some(Token::LetExpr(var, val)));
342        // inline fmt calls
343        } else if curr_tok == "$" && self.peek()? == "(" {
344            self.advance_char()?;
345            self.advance_char()?;
346            self.must_consume(")")?;
347            let fmt = self.consume_till(")")?.to_string();
348
349            return Ok(Some(Token::Fn(FnKind::Fmt, fmt)));
350        // fmt calls
351        } else if curr_tok == "f" && self.peek()? == "m" && self.peek_n(2)? == "t" {
352            self.advance_char()?;
353            self.advance_char()?;
354            self.advance_char()?;
355            self.must_consume("(")?;
356
357            let mut fmt = String::new();
358            if self.char()? != ")" {
359                // the body expression may have parenthesis in it, so need to maintain a stack and
360                // consume until the stack is empty
361                fmt = self.consume_nested_parenthesis()?.trim().to_string();
362                // remove the last character
363                fmt.pop();
364            }
365
366            self.must_consume(")")?;
367
368            return Ok(Some(Token::Fn(FnKind::Fmt, fmt)));
369        // eval calls
370        } else if curr_tok == "e"
371            && self.peek()? == "v"
372            && self.peek_n(2)? == "a"
373            && self.peek_n(3)? == "l"
374        {
375            self.advance_char()?;
376            self.advance_char()?;
377            self.advance_char()?;
378            self.advance_char()?;
379            self.must_consume("(")?;
380
381            let mut eval = String::new();
382            if self.char()? != ")" {
383                eval = self.consume_nested_parenthesis()?.trim().to_string();
384                // remove the last character
385                eval.pop();
386            }
387
388            self.must_consume(")")?;
389
390            return Ok(Some(Token::Fn(FnKind::Eval, eval)));
391        // headers
392        } else if curr_tok == "#" {
393            let hash_count = self.consume_until_not("#")?.len();
394
395            let heading = self.consume_line()?;
396            let heading = heading.trim();
397
398            let header_kind: HeaderKind = hash_count.into();
399
400            return Ok(Some(Token::Markdown(MarkdownTag::Header(
401                header_kind,
402                heading.to_string(),
403            ))));
404        // blockquote
405        } else if curr_tok == ">" {
406            self.advance_char()?;
407            // only a blockquote if previously it was a newline
408            if self.prv.is_some() && self.prv.clone().unwrap() == "\n" {
409                let blockquote = self.consume_line()?;
410                let blockquote = blockquote.trim();
411                return Ok(Some(Token::Markdown(MarkdownTag::Blockquote(
412                    blockquote.to_string(),
413                ))));
414            // this is to ensure an arrow like this, "->" can be made in text
415            } else {
416                return Ok(Some(Token::Text(None, String::from(">"))));
417            }
418
419        // bullets or checkboxes
420        } else if curr_tok == "-" {
421            self.advance_char()?;
422            self.consume_whitespace()?;
423
424            let mut is_bullet = self.char()? != "[";
425
426            // only a bullet if the next character is an ascii alphabet number
427            let is_next_alnum = self.peek()?.chars().next().unwrap().is_ascii_alphanumeric();
428            is_bullet = is_bullet && is_next_alnum;
429
430            if is_bullet {
431                let bullet = self.consume_line()?;
432                let bullet = bullet.trim();
433                return Ok(Some(Token::Markdown(MarkdownTag::BulletPoint(
434                    bullet.to_string(),
435                ))));
436            }
437
438            let is_checkbox = self.char()? == "[";
439            if is_checkbox {
440                self.advance_char()?;
441                let is_checked = self.char()? == "x";
442
443                self.advance_char()?;
444                self.must_consume("]")?;
445                self.consume_whitespace()?;
446
447                let checkbox = self.consume_line()?;
448                let checkbox = checkbox.trim();
449                return Ok(Some(Token::Markdown(MarkdownTag::Checkbox(
450                    is_checked,
451                    checkbox.to_string(),
452                ))));
453            }
454
455            return Ok(Some(Token::Text(None, curr_tok.to_string())));
456
457        // line separator
458        } else if curr_tok == "=" && self.peek()? == "=" && self.peek_n(2)? == "=" {
459            let prev = self.pos;
460            self.consume_until_not("=")?;
461            let now = self.pos;
462            if (now - prev == 3) && !self.consume_line()?.trim().is_empty() {
463                let e = ErrorKind::GrammarGoblin(
464                    "Line separator should contain only '=' characters".to_string(),
465                );
466                self.create_error(e);
467                return Err(self.ctx.clone());
468            }
469
470            return Ok(Some(Token::Markdown(MarkdownTag::LineSeparator)));
471        // consume links
472        } else if (curr_tok == "!" && self.peek()? == "[") || curr_tok == "[" {
473            let is_image = curr_tok == "!";
474            if is_image {
475                self.advance_char()?;
476            }
477            self.must_consume("[")?;
478            let text = self.consume_till("]")?.to_string();
479            self.must_consume("]")?;
480            self.must_consume("(")?;
481            let link = self.consume_till(")")?.to_string();
482            self.must_consume(")")?;
483
484            return Ok(Some(Token::Markdown(MarkdownTag::Link(
485                if is_image {
486                    LinkKind::Image
487                } else {
488                    LinkKind::Hyperlink
489                },
490                text,
491                link,
492            ))));
493        // code blocks
494        } else if curr_tok == "`" {
495            // check if inline code block or code block
496            let code: String;
497            if self.peek()? == "`" {
498                self.must_consume("`")?;
499                self.must_consume("`")?;
500                self.must_consume("`")?;
501
502                self.consume_whitespace()?;
503                code = self.consume_till("`")?.to_string();
504
505                self.must_consume("`")?;
506                self.must_consume("`")?;
507                self.must_consume("`")?;
508            } else {
509                self.must_consume("`")?;
510                code = self.consume_till("`")?.trim().to_string();
511                self.must_consume("`")?;
512            }
513
514            return Ok(Some(Token::Markdown(MarkdownTag::CodeBlock(code))));
515        // bold
516        } else if curr_tok == "*" {
517            if self.peek()? == "*" {
518                self.advance_char()?;
519                self.advance_char()?;
520                let text = self.consume_till("*")?.to_string();
521                self.must_consume("*")?;
522                self.must_consume("*")?;
523
524                return Ok(Some(Token::Text(Some(Emphasis::Bold), text)));
525            } else {
526                self.advance_char()?;
527                let text = self.consume_till("*")?.to_string();
528                self.must_consume("*")?;
529
530                return Ok(Some(Token::Text(Some(Emphasis::Italic), text)));
531            }
532        // strikethrough
533        } else if curr_tok == "~" {
534            self.advance_char()?;
535            let text = self.consume_till("~")?.to_string();
536            self.must_consume("~")?;
537
538            return Ok(Some(Token::Text(Some(Emphasis::Strikethrough), text)));
539        // text
540        } else {
541            let text = curr_tok.to_string();
542            self.advance_char()?;
543            return Ok(Some(Token::Text(None, text)));
544        }
545    }
546
547    /// Takes in a collection of tokens and tries
548    /// to compacts the repeated text tokens into one
549    /// only if they have the same emphasis
550    pub fn compact(tokens: Vec<Token>) -> Vec<Token> {
551        let mut compacted = Vec::new();
552        let mut iter = tokens.iter().peekable();
553
554        while let Some(token) = iter.next() {
555            match token {
556                Token::Text(emphasis, text) => {
557                    let mut combined_text = text.clone();
558                    while let Some(&Token::Text(next_emphasis, next_text)) = iter.peek() {
559                        if emphasis == next_emphasis {
560                            combined_text.push_str(next_text);
561                            iter.next(); // consume the token
562                        } else {
563                            break;
564                        }
565                    }
566                    compacted.push(Token::Text(emphasis.clone(), combined_text));
567                }
568                _ => compacted.push(token.clone()),
569            }
570        }
571        compacted
572    }
573}
mazer_core/tokenizer.rs

mazer_core/
tokenizer.rs
