1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
use std::convert::From;

pub type Spanned<Tok, Loc, Error> = Result<(Loc, Tok, Loc), Error>;

#[derive(Copy, Clone, Debug)]
pub enum Tok<'input> {
    Symbol(&'input str),
    CurlyOpen,
    CurlyClose,
    Space,
    Linefeed,
}

impl<'input> From<Tok<'input>> for String {
    fn from(t: Tok<'input>) -> String {
        match t {
            Tok::Linefeed => String::from("\n"),
            Tok::CurlyClose => String::from("}"),
            Tok::CurlyOpen => String::from("{"),
            Tok::Space => String::from(" "),
            Tok::Symbol(s) => String::from(s),
        }
    }
}

use std::str::CharIndices;

pub struct Lexer<'input> {
    chars: std::iter::Peekable<CharIndices<'input>>,
    input: &'input str,
}

impl<'input> Lexer<'input> {
    pub fn new(input: &'input str) -> Self {
        Lexer {
            chars: input.char_indices().peekable(),
            input,
        }
    }
}

impl<'input> Iterator for Lexer<'input> {
    type Item = Spanned<Tok<'input>, usize, ()>;

    fn next(&mut self) -> Option<Self::Item> {
        loop {
            match self.chars.next() {
                Some((_, '\t')) => continue, // Tabs not supported
                Some((i, ' ')) => return Some(Ok((i, Tok::Space, i + 1))),
                Some((i, '\n')) => return Some(Ok((i, Tok::Linefeed, i + 1))),
                Some((i, '}')) => return Some(Ok((i, Tok::CurlyClose, i + 1))),
                Some((i, '{')) => return Some(Ok((i, Tok::CurlyOpen, i + 1))),

                None => return None, // End of file
                Some((i, _)) => loop {
                    match self.chars.peek() {
                        Some((j, '}')) | Some((j, '{')) | Some((j, '\n')) | Some((j, ' ')) => {
                            return Some(Ok((i, Tok::Symbol(&self.input[i..*j]), *j)))
                        }
                        None => {
                            return Some(Ok((
                                i,
                                Tok::Symbol(&self.input[i..]),
                                self.input.len(),
                            )))
                        }
                        _ => {}
                    }
                    self.chars.next();
                },
            }
        }
    }
}