yozuk_helper_preprocessor/
lib.rs

1#![forbid(unsafe_code)]
2#![deny(clippy::all)]
3
4use std::collections::VecDeque;
5use std::fmt;
6use yozuk_sdk::prelude::*;
7
8pub trait TokenParser: Send + Sync {
9    fn parse(&self, tokens: &[Token]) -> Option<Token>;
10}
11
12pub struct TokenMerger<P> {
13    parser: P,
14}
15
16impl<P> fmt::Debug for TokenMerger<P> {
17    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
18        f.debug_struct("TokenMerger").finish()
19    }
20}
21
22impl<P> TokenMerger<P> {
23    pub fn new(parser: P) -> Self {
24        Self { parser }
25    }
26}
27
28impl<P> Preprocessor for TokenMerger<P>
29where
30    P: TokenParser + 'static,
31{
32    fn preprocess(&self, input: Vec<Token>) -> Vec<Token> {
33        let mut output = Vec::new();
34        let mut tokens = input.into_iter().collect::<VecDeque<_>>();
35        while !tokens.is_empty() {
36            for i in 1..=tokens.len() {
37                let len = tokens.len() + 1 - i;
38                let exp = tokens.as_slices().0;
39                let exp = if exp.len() >= len {
40                    &exp[..len]
41                } else {
42                    &tokens.make_contiguous()[..len]
43                };
44                if let Some(merged) = self.parser.parse(exp) {
45                    for _ in 0..len {
46                        tokens.pop_front();
47                    }
48                    output.push(merged);
49                    break;
50                }
51            }
52            if let Some(front) = tokens.pop_front() {
53                output.push(front);
54            }
55        }
56        output
57    }
58}
yozuk_helper_preprocessor/lib.rs

yozuk_helper_preprocessor/
lib.rs
