yozuk_sdk/
preprocessor.rs

1#![forbid(unsafe_code)]
2#![deny(clippy::all)]
3
4use crate::skill::*;
5use crate::token::*;
6use std::collections::VecDeque;
7
8pub trait TokenParser: Send + Sync {
9    fn parse(&self, tokens: &[Token]) -> Option<Token>;
10}
11
12pub struct TokenMerger<P> {
13    parser: P,
14}
15
16impl<P> TokenMerger<P> {
17    pub fn new(parser: P) -> Self {
18        Self { parser }
19    }
20}
21
22impl<P> Preprocessor for TokenMerger<P>
23where
24    P: TokenParser + 'static,
25{
26    fn preprocess(&self, input: Vec<Token>) -> Vec<Token> {
27        let mut output = Vec::new();
28        let mut tokens = input.into_iter().collect::<VecDeque<_>>();
29        while !tokens.is_empty() {
30            for i in 1..=tokens.len() {
31                let len = tokens.len() + 1 - i;
32                let exp = tokens.as_slices().0;
33                let exp = if exp.len() >= len {
34                    &exp[..len]
35                } else {
36                    &tokens.make_contiguous()[..len]
37                };
38                if let Some(merged) = self.parser.parse(exp) {
39                    for _ in 0..len {
40                        tokens.pop_front();
41                    }
42                    output.push(merged);
43                    break;
44                }
45            }
46            if let Some(front) = tokens.pop_front() {
47                output.push(front);
48            }
49        }
50        output
51    }
52}
yozuk_sdk/preprocessor.rs

yozuk_sdk/
preprocessor.rs
