ironwork_syntax/
lexer.rs

1use crate::source::Source;
2use crate::{Error, Pos};
3
4#[derive(Clone, Debug, PartialEq, Eq)]
5pub enum Tok {
6    /// A COBOL word, uppercased: a name, a reserved word or a level number with letters in it.
7    Word(String),
8    /// A numeric literal as written: optional sign, digits, optional decimal point.
9    Number(String),
10    Alnum(String),
11    Hex(Vec<u8>),
12    National(String),
13    Pic(String),
14    /// An EXEC ... END-EXEC block, as written: SQL, CICS or DLI for a precompiler.
15    Exec(String),
16    Period,
17    LParen,
18    RParen,
19    Colon,
20    Plus,
21    Minus,
22    Star,
23    Slash,
24    Power,
25    Eq,
26    Lt,
27    Gt,
28    Le,
29    Ge,
30}
31
32#[derive(Clone, Debug, PartialEq, Eq)]
33pub struct Token {
34    pub tok: Tok,
35    pub pos: Pos,
36    /// Starts in area A (columns 8 to 11), where division, section and paragraph headers go.
37    pub area_a: bool,
38}
39
40struct Lexer<'a> {
41    chars: Vec<char>,
42    positions: &'a [Pos],
43    at: usize,
44    tokens: Vec<Token>,
45}
46
47pub fn lex(source: &Source) -> Result<Vec<Token>, Error> {
48    let mut lx = Lexer { chars: source.text.chars().collect(), positions: &source.positions, at: 0, tokens: Vec::new() };
49    while lx.at < lx.chars.len() {
50        lx.next_token()?;
51    }
52    Ok(lx.tokens)
53}
54
55fn is_word_char(c: char) -> bool {
56    c.is_ascii_alphanumeric() || c == '-' || c == '_'
57}
58
59impl Lexer<'_> {
60    fn peek(&self, ahead: usize) -> Option<char> {
61        self.chars.get(self.at + ahead).copied()
62    }
63
64    fn pos(&self) -> Pos {
65        self.positions.get(self.at).copied().unwrap_or_default()
66    }
67
68    fn separator_follows(&self, ahead: usize) -> bool {
69        self.peek(ahead).is_none_or(|c| c == ' ' || c == '\n')
70    }
71
72    fn emit(&mut self, tok: Tok, pos: Pos) {
73        self.tokens.push(Token { tok, pos, area_a: (8..=11).contains(&pos.col) });
74    }
75
76    fn expecting_picture(&self) -> bool {
77        let words: Vec<&str> =
78            self.tokens.iter().rev().take(2).map(|t| if let Tok::Word(w) = &t.tok { w.as_str() } else { "" }).collect();
79        matches!(words.as_slice(), ["PIC" | "PICTURE", ..] | ["IS", "PIC" | "PICTURE"])
80    }
81
82    fn next_token(&mut self) -> Result<(), Error> {
83        let c = self.chars[self.at];
84        let pos = self.pos();
85        if c == ' ' || c == '\n' || c == ',' || c == ';' {
86            self.at += 1;
87            return Ok(());
88        }
89        if self.expecting_picture() {
90            return self.picture(pos);
91        }
92        let next = self.peek(1);
93        let quote_next = matches!(next, Some('\'' | '"'));
94        match c {
95            '\'' | '"' => {
96                let text = self.quoted(pos)?;
97                self.emit(Tok::Alnum(text), pos);
98            }
99            'X' | 'x' if quote_next => {
100                self.at += 1;
101                let text = self.quoted(pos)?;
102                let bytes = unhex(&text).ok_or_else(|| Error::at(pos, format!("X'{text}' is not an even number of hex digits")))?;
103                self.emit(Tok::Hex(bytes), pos);
104            }
105            'N' | 'n' if quote_next => {
106                self.at += 1;
107                let text = self.quoted(pos)?;
108                self.emit(Tok::National(text), pos);
109            }
110            '.' if self.separator_follows(1) => {
111                self.at += 1;
112                self.emit(Tok::Period, pos);
113            }
114            '+' | '-' if next.is_some_and(|n| n.is_ascii_digit() || (n == '.' && self.peek(2).is_some_and(|d| d.is_ascii_digit()))) => {
115                self.at += 1;
116                let digits = self.number_or_word(pos)?;
117                match digits {
118                    Tok::Number(n) => self.emit(Tok::Number(format!("{c}{n}")), pos),
119                    _ => return Err(Error::at(pos, "a sign must be followed by a number")),
120                }
121            }
122            _ if c.is_ascii_alphanumeric() || c == '.' => {
123                let tok = self.number_or_word(pos)?;
124                match tok {
125                    Tok::Word(w) if w == "EXEC" || w == "EXECUTE" => {
126                        let tok = self.exec_block(pos)?;
127                        self.emit(tok, pos);
128                    }
129                    tok => self.emit(tok, pos),
130                }
131            }
132            _ => {
133                let (tok, len) = match (c, next) {
134                    ('*', Some('*')) => (Tok::Power, 2),
135                    ('<', Some('=')) => (Tok::Le, 2),
136                    ('>', Some('=')) => (Tok::Ge, 2),
137                    ('*', _) => (Tok::Star, 1),
138                    ('/', _) => (Tok::Slash, 1),
139                    ('+', _) => (Tok::Plus, 1),
140                    ('-', _) => (Tok::Minus, 1),
141                    ('=', _) => (Tok::Eq, 1),
142                    ('<', _) => (Tok::Lt, 1),
143                    ('>', _) => (Tok::Gt, 1),
144                    ('(', _) => (Tok::LParen, 1),
145                    (')', _) => (Tok::RParen, 1),
146                    (':', _) => (Tok::Colon, 1),
147                    _ => return Err(Error::at(pos, format!("unexpected character {c:?}"))),
148                };
149                self.at += len;
150                self.emit(tok, pos);
151            }
152        }
153        Ok(())
154    }
155
156    /// The text of an EXEC block up to END-EXEC, which is consumed; quotes inside are skipped whole.
157    fn exec_block(&mut self, pos: Pos) -> Result<Tok, Error> {
158        let start = self.at;
159        let mut quote: Option<char> = None;
160        while let Some(c) = self.peek(0) {
161            match quote {
162                Some(q) if c == q => quote = None,
163                Some(_) => {}
164                None if c == '\'' || c == '"' => quote = Some(c),
165                None if c.eq_ignore_ascii_case(&'E') => {
166                    let ahead: String = self.chars[self.at..].iter().take(8).collect();
167                    let boundary = self.chars.get(self.at + 8).is_none_or(|d| !is_word_char(*d)) && (self.at == 0 || !is_word_char(self.chars[self.at - 1]));
168                    if ahead.eq_ignore_ascii_case("END-EXEC") && boundary {
169                        let text: String = self.chars[start..self.at].iter().collect();
170                        self.at += 8;
171                        return Ok(Tok::Exec(text.split_whitespace().collect::<Vec<_>>().join(" ")));
172                    }
173                }
174                None => {}
175            }
176            self.at += 1;
177        }
178        Err(Error::at(pos, "EXEC with no END-EXEC"))
179    }
180
181    /// Reads a quoted literal starting at the opening quote; a doubled quote stands for one.
182    fn quoted(&mut self, pos: Pos) -> Result<String, Error> {
183        let quote = self.chars[self.at];
184        self.at += 1;
185        let mut text = String::new();
186        loop {
187            match self.peek(0) {
188                None | Some('\n') => return Err(Error::at(pos, "an unterminated literal")),
189                Some(c) if c == quote && self.peek(1) == Some(quote) => {
190                    text.push(quote);
191                    self.at += 2;
192                }
193                Some(c) if c == quote => {
194                    self.at += 1;
195                    return Ok(text);
196                }
197                Some(c) => {
198                    text.push(c);
199                    self.at += 1;
200                }
201            }
202        }
203    }
204
205    fn number_or_word(&mut self, pos: Pos) -> Result<Tok, Error> {
206        let start = self.at;
207        while self.peek(0).is_some_and(is_word_char) {
208            self.at += 1;
209        }
210        let run: String = self.chars[start..self.at].iter().collect();
211        let all_digits = run.chars().all(|c| c.is_ascii_digit());
212        if all_digits && self.peek(0) == Some('.') && self.peek(1).is_some_and(|c| c.is_ascii_digit()) {
213            self.at += 1;
214            let frac_start = self.at;
215            while self.peek(0).is_some_and(|c| c.is_ascii_digit()) {
216                self.at += 1;
217            }
218            let frac: String = self.chars[frac_start..self.at].iter().collect();
219            return Ok(Tok::Number(format!("{run}.{frac}")));
220        }
221        if run.is_empty() {
222            return Err(Error::at(pos, "unexpected '.'"));
223        }
224        Ok(if all_digits { Tok::Number(run) } else { Tok::Word(run.to_ascii_uppercase()) })
225    }
226
227    fn picture(&mut self, pos: Pos) -> Result<(), Error> {
228        let start = self.at;
229        while self.peek(0).is_some_and(|c| c != ' ' && c != '\n') {
230            self.at += 1;
231        }
232        let mut end = self.at;
233        while end > start && matches!(self.chars[end - 1], '.' | ',' | ';') {
234            end -= 1;
235        }
236        let text: String = self.chars[start..end].iter().collect();
237        if text.is_empty() {
238            return Err(Error::at(pos, "PICTURE with no character-string"));
239        }
240        self.at = end;
241        let text = text.to_ascii_uppercase();
242        let tok = if text == "IS" && self.tokens.last().is_some_and(|t| matches!(&t.tok, Tok::Word(w) if w == "PIC" || w == "PICTURE")) {
243            Tok::Word(text)
244        } else {
245            Tok::Pic(text)
246        };
247        self.emit(tok, pos);
248        Ok(())
249    }
250}
251
252fn unhex(text: &str) -> Option<Vec<u8>> {
253    if !text.len().is_multiple_of(2) || !text.is_ascii() {
254        return None;
255    }
256    (0..text.len()).step_by(2).map(|i| u8::from_str_radix(&text[i..i + 2], 16).ok()).collect()
257}
258
259#[cfg(test)]
260mod tests {
261    use super::*;
262    use crate::source;
263
264    fn toks(text: &str) -> Vec<Tok> {
265        lex(&source::read(text).unwrap()).unwrap().into_iter().map(|t| t.tok).collect()
266    }
267
268    fn w(s: &str) -> Tok {
269        Tok::Word(s.into())
270    }
271
272    #[test]
273    fn numbers_words_and_the_separator_period() {
274        assert_eq!(toks("           05 A-1 VALUE 0.1."), [Tok::Number("05".into()), w("A-1"), w("VALUE"), Tok::Number("0.1".into()), Tok::Period]);
275        assert_eq!(toks("           VALUE -12345."), [w("VALUE"), Tok::Number("-12345".into()), Tok::Period]);
276        assert_eq!(toks("       100-MAIN."), [w("100-MAIN"), Tok::Period]);
277    }
278
279    #[test]
280    fn operators_need_spaces_and_signed_literals_do_not() {
281        assert_eq!(toks("           A - 1 ** 2"), [w("A"), Tok::Minus, Tok::Number("1".into()), Tok::Power, Tok::Number("2".into())]);
282        assert_eq!(toks("           >= <= ("), [Tok::Ge, Tok::Le, Tok::LParen]);
283    }
284
285    #[test]
286    fn literals() {
287        assert_eq!(toks("           'IT''S' X'F1C1' N'AB'"), [Tok::Alnum("IT'S".into()), Tok::Hex(vec![0xF1, 0xC1]), Tok::National("AB".into())]);
288    }
289
290    #[test]
291    fn a_picture_is_one_token_and_keeps_its_own_periods() {
292        assert_eq!(toks("           PIC S9(3)V99 COMP-3."), [w("PIC"), Tok::Pic("S9(3)V99".into()), w("COMP-3"), Tok::Period]);
293        assert_eq!(toks("           PICTURE IS ZZ,ZZ9.99."), [w("PICTURE"), w("IS"), Tok::Pic("ZZ,ZZ9.99".into()), Tok::Period]);
294    }
295
296    #[test]
297    fn commas_between_operands_are_separators() {
298        assert_eq!(toks("           F(A, 1)"), [w("F"), Tok::LParen, w("A"), Tok::Number("1".into()), Tok::RParen]);
299        assert_eq!(toks("           T(I,J)"), [w("T"), Tok::LParen, w("I"), w("J"), Tok::RParen]);
300    }
301
302    #[test]
303    fn an_exec_block_is_one_token() {
304        assert_eq!(
305            toks("           EXEC SQL SELECT A.B INTO :X FROM T WHERE C = 'END-EXEC'\n               END-EXEC."),
306            [Tok::Exec("SQL SELECT A.B INTO :X FROM T WHERE C = 'END-EXEC'".into()), Tok::Period]
307        );
308    }
309
310    #[test]
311    fn area_a_is_marked() {
312        let t = lex(&source::read("       PARA.\n           MOVE").unwrap()).unwrap();
313        assert!(t[0].area_a);
314        assert!(!t[2].area_a);
315    }
316}
ironwork_syntax/lexer.rs

ironwork_syntax/
lexer.rs
