tokay 0.5.0

Tokay is a programming language designed for ad-hoc parsing, inspired by awk.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

use super::*;

/** Alternation construct.

The alternation construct defines either an alternation of sequences or a grouped sequence
of instructions. An alternation is only performed when input is consumed, otherwise the
alternation works similar to a sequence of sequences.
*/

#[derive(Debug)]
pub struct ImlAlternation {
    items: Vec<ImlOp>,
}

impl ImlAlternation {
    pub fn new(items: Vec<ImlOp>) -> ImlOp {
        Self { items: items }.into_op()
    }
}

impl Compileable for ImlAlternation {
    fn resolve(&mut self, usages: &mut Vec<Vec<ImlOp>>) {
        for alt in self.items.iter_mut() {
            alt.resolve(usages);
        }
    }

    fn finalize(
        &mut self,
        values: &Vec<ImlValue>,
        stack: &mut Vec<(usize, bool)>,
    ) -> Option<Consumable> {
        let mut leftrec = false;
        let mut nullable = false;
        let mut consumes = false;

        for alt in self.items.iter_mut() {
            if let Some(consumable) = alt.finalize(values, stack) {
                leftrec |= consumable.leftrec;
                nullable |= consumable.nullable;
                consumes = true;
            }
        }

        if consumes {
            Some(Consumable { leftrec, nullable })
        } else {
            None
        }
    }

    fn compile(&self, parselet: &ImlParselet) -> Vec<Op> {
        let mut ret = Vec::new();
        let mut iter = self.items.iter();
        let mut jumps = Vec::new();

        while let Some(item) = iter.next() {
            let alt = item.compile(parselet);

            if iter.len() > 0 {
                ret.push(Op::Fuse(alt.len() + 3));
                ret.extend(alt);
                ret.push(Op::Nop);
                ret.push(Op::Reset);

                jumps.push(ret.len() - 2);
            } else {
                ret.extend(alt);
            }
        }

        while let Some(addr) = jumps.pop() {
            ret[addr] = Op::ForwardIfConsumed(ret.len() - addr);
        }

        if self.items.len() > 1 {
            ret.insert(0, Op::Frame(0));
            ret.push(Op::Close);
        }

        ret
    }
}

impl std::fmt::Display for ImlAlternation {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{{")?;
        for item in &self.items {
            write!(f, "{} ", item)?;
        }
        write!(f, "}}")?;

        Ok(())
    }
}