tokenizer_lib/
lib.rs

1#![doc = include_str!("./README.md")]
2#![allow(clippy::type_complexity, clippy::new_ret_no_self)]
3
4use std::{
5    collections::VecDeque,
6    fmt::{self, Debug},
7    usize,
8};
9
10#[cfg(feature = "buffered")]
11pub use buffered_token_queue::*;
12#[cfg(feature = "generator")]
13pub use generator_token_queue::*;
14#[cfg(all(not(target_arch = "wasm32"), feature = "parallel"))]
15pub use parallel_token_queue::*;
16
17/// [PartialEq] is required for comparing tokens with [TokenReader::expect_next]
18pub trait TokenTrait: PartialEq {
19    /// Use this for *nully* tokens. Will be skipped under [TokenReader::expect_next]
20    fn is_skippable(&self) -> bool {
21        false
22    }
23}
24
25/// A structure with a piece of data and some additional data such as a position
26#[cfg_attr(any(test, doctest), derive(PartialEq, Eq))]
27pub struct Token<T: TokenTrait, TData>(pub T, pub TData);
28
29impl<T: TokenTrait + Debug, TData: Debug> Debug for Token<T, TData> {
30    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
31        f.debug_tuple("Token")
32            .field(&self.0)
33            .field(&self.1)
34            .finish()
35    }
36}
37
38/// A *reader* over a sequence of tokens
39pub trait TokenReader<T: TokenTrait, TData> {
40    /// Returns a reference to next token but does not advance current position
41    fn peek(&mut self) -> Option<&Token<T, TData>>;
42
43    /// Returns a reference to nth (zero based) upcoming token without advancing
44    fn peek_n(&mut self, n: usize) -> Option<&Token<T, TData>>;
45
46    /// Use with caution
47    fn peek_mut(&mut self) -> Option<&mut Token<T, TData>>;
48
49    /// Returns the next token and advances
50    fn next(&mut self) -> Option<Token<T, TData>>;
51
52    /// Returns next if `cb` returns true for the upcoming token (the token from [TokenReader::peek])
53    fn conditional_next(&mut self, cb: impl FnOnce(&T) -> bool) -> Option<Token<T, TData>> {
54        let peek = self.peek()?;
55        if cb(&peek.0) {
56            self.next()
57        } else {
58            None
59        }
60    }
61
62    /// Runs the closure (cb) over upcoming tokens. Passes the value behind the Token to the closure.
63    /// Will stop and return a reference **to the next Token from when the closure returns true**.
64    /// Returns None if scanning finishes before closure returns true. Does not advance the reader.
65    ///
66    /// Used for lookahead and then branching based on return value during parsing
67    fn scan(&mut self, cb: impl FnMut(&T, &TData) -> bool) -> Option<&Token<T, TData>>;
68
69    /// Tests that next token matches an expected type. Will return error if does not
70    /// match. The `Ok` value contains the data of the valid token.
71    /// Else it will return the Err with the expected token type and the token that did not match
72    ///
73    /// Is the token is skippable (using [TokenTrait::is_skippable])
74    fn expect_next(&mut self, expected_type: T) -> Result<TData, Option<(T, Token<T, TData>)>> {
75        match self.next() {
76            Some(token) => {
77                if token.0 == expected_type {
78                    Ok(token.1)
79                } else if token.0.is_skippable() {
80                    // This will advance to the next, won't cyclically recurse
81                    self.expect_next(expected_type)
82                } else {
83                    Err(Some((expected_type, token)))
84                }
85            }
86            None => Err(None),
87        }
88    }
89}
90
91/// Trait for a sender that can append a token to a sequence
92pub trait TokenSender<T: TokenTrait, TData> {
93    /// Appends a new [`Token`]
94    /// Will return false if could not push token
95    fn push(&mut self, token: Token<T, TData>) -> bool;
96}
97
98#[cfg(feature = "buffered")]
99mod buffered_token_queue {
100    use super::*;
101    /// A queue which can be used as a sender and reader. Use this for buffering all the tokens before reading
102    pub struct BufferedTokenQueue<T: TokenTrait, TData> {
103        buffer: VecDeque<Token<T, TData>>,
104    }
105
106    impl<T: TokenTrait, TData> Default for BufferedTokenQueue<T, TData> {
107        fn default() -> Self {
108            Self {
109                buffer: Default::default(),
110            }
111        }
112    }
113
114    impl<T: TokenTrait, TData> BufferedTokenQueue<T, TData> {
115        /// Constructs a new [`BufferedTokenQueue`]
116        pub fn new() -> Self {
117            Default::default()
118        }
119    }
120
121    impl<T: TokenTrait, TData> TokenSender<T, TData> for BufferedTokenQueue<T, TData> {
122        fn push(&mut self, token: Token<T, TData>) -> bool {
123            self.buffer.push_back(token);
124            true
125        }
126    }
127
128    impl<T: TokenTrait, TData> TokenReader<T, TData> for BufferedTokenQueue<T, TData> {
129        fn peek(&mut self) -> Option<&Token<T, TData>> {
130            self.buffer.front()
131        }
132
133        fn peek_n(&mut self, n: usize) -> Option<&Token<T, TData>> {
134            self.buffer.get(n)
135        }
136
137        fn peek_mut(&mut self) -> Option<&mut Token<T, TData>> {
138            self.buffer.front_mut()
139        }
140
141        fn next(&mut self) -> Option<Token<T, TData>> {
142            self.buffer.pop_front()
143        }
144
145        fn scan(&mut self, mut cb: impl FnMut(&T, &TData) -> bool) -> Option<&Token<T, TData>> {
146            let mut iter = self.buffer.iter().peekable();
147            while let Some(token) = iter.next() {
148                if cb(&token.0, &token.1) {
149                    return iter.peek().copied();
150                }
151            }
152            None
153        }
154    }
155}
156
157#[cfg(all(not(target_arch = "wasm32"), feature = "parallel"))]
158mod parallel_token_queue {
159    use super::*;
160    use std::sync::mpsc::{sync_channel, Receiver, RecvError, SyncSender};
161
162    const DEFAULT_BUFFER_SIZE: usize = 20;
163
164    /// A token queue used for doing lexing and parsing on different threads. Will send tokens between threads
165    pub struct ParallelTokenQueue;
166
167    impl ParallelTokenQueue {
168        /// Creates two items, a sender and a receiver. Where the reader is on the parsing thread and the
169        /// sender is on the lexer thread
170        pub fn new<T: TokenTrait, TData>(
171        ) -> (ParallelTokenSender<T, TData>, ParallelTokenReader<T, TData>) {
172            Self::new_with_buffer_size(DEFAULT_BUFFER_SIZE)
173        }
174
175        pub fn new_with_buffer_size<T: TokenTrait, TData>(
176            buffer_size: usize,
177        ) -> (ParallelTokenSender<T, TData>, ParallelTokenReader<T, TData>) {
178            let (sender, receiver) = sync_channel::<Token<T, TData>>(buffer_size);
179
180            (
181                ParallelTokenSender(sender),
182                ParallelTokenReader {
183                    receiver,
184                    cache: VecDeque::new(),
185                },
186            )
187        }
188    }
189
190    // Sender and reader structs generate by `ParallelTokenQueue::new`:
191
192    #[doc(hidden)]
193    pub struct ParallelTokenSender<T: TokenTrait, TData>(SyncSender<Token<T, TData>>);
194
195    #[doc(hidden)]
196    pub struct ParallelTokenReader<T: TokenTrait, TData> {
197        receiver: Receiver<Token<T, TData>>,
198        cache: VecDeque<Token<T, TData>>,
199    }
200
201    impl<T: TokenTrait, TData> TokenSender<T, TData> for ParallelTokenSender<T, TData> {
202        fn push(&mut self, token: Token<T, TData>) -> bool {
203            self.0.send(token).is_ok()
204        }
205    }
206
207    impl<T: TokenTrait, TData> TokenReader<T, TData> for ParallelTokenReader<T, TData> {
208        fn peek(&mut self) -> Option<&Token<T, TData>> {
209            if self.cache.is_empty() {
210                match self.receiver.recv() {
211                    Ok(token) => self.cache.push_back(token),
212                    // Err is reader has dropped e.g. no more tokens
213                    Err(RecvError) => {
214                        return None;
215                    }
216                }
217            }
218            self.cache.front()
219        }
220
221        fn peek_n(&mut self, n: usize) -> Option<&Token<T, TData>> {
222            while self.cache.len() <= n {
223                match self.receiver.recv() {
224                    Ok(token) => self.cache.push_back(token),
225                    // Err is reader has dropped e.g. no more tokens
226                    Err(RecvError) => {
227                        return None;
228                    }
229                }
230            }
231            self.cache.get(n)
232        }
233
234        fn next(&mut self) -> Option<Token<T, TData>> {
235            if !self.cache.is_empty() {
236                return self.cache.pop_front();
237            }
238            self.receiver.recv().ok()
239        }
240
241        fn scan(&mut self, mut cb: impl FnMut(&T, &TData) -> bool) -> Option<&Token<T, TData>> {
242            let found = scan_cache(&mut self.cache, &mut cb);
243            let mut return_next = match found {
244                ScanCacheResult::RetrievableInCacheAt(idx) => return self.cache.get(idx),
245                ScanCacheResult::Found => true,
246                ScanCacheResult::NotFound => false,
247            };
248            loop {
249                match self.receiver.recv() {
250                    Ok(val) => {
251                        if return_next {
252                            self.cache.push_back(val);
253                            return self.cache.back();
254                        }
255                        if cb(&val.0, &val.1) {
256                            return_next = true;
257                        }
258                        self.cache.push_back(val);
259                    }
260                    // Err is reader has dropped e.g. no more tokens
261                    Err(RecvError) => {
262                        return None;
263                    }
264                }
265            }
266        }
267
268        fn peek_mut(&mut self) -> Option<&mut Token<T, TData>> {
269            if self.cache.is_empty() {
270                match self.receiver.recv() {
271                    Ok(token) => self.cache.push_back(token),
272                    // Err is reader has dropped e.g. no more tokens
273                    Err(RecvError) => {
274                        return None;
275                    }
276                }
277            }
278            self.cache.front_mut()
279        }
280    }
281}
282
283#[cfg(feature = "generator")]
284mod generator_token_queue {
285    use super::*;
286
287    /// A token queue which has a backing generator/lexer which is called when needed by parsing logic
288    pub struct GeneratorTokenQueue<T, TData, TGeneratorState, TGenerator>
289    where
290        T: TokenTrait,
291        for<'a> TGenerator:
292            FnMut(&mut TGeneratorState, &mut GeneratorTokenQueueBuffer<'a, T, TData>),
293    {
294        generator: TGenerator,
295        generator_state: TGeneratorState,
296        cache: VecDeque<Token<T, TData>>,
297    }
298
299    /// A wrapping struct for the cache around [`GeneratorTokenQueue`]. Use as the second parameter
300    /// in the generator/lexer function
301    pub struct GeneratorTokenQueueBuffer<'a, T: TokenTrait, TData>(
302        &'a mut VecDeque<Token<T, TData>>,
303    );
304
305    impl<'a, T: TokenTrait, TData> GeneratorTokenQueueBuffer<'a, T, TData> {
306        pub fn is_empty(&self) -> bool {
307            self.0.is_empty()
308        }
309
310        pub fn len(&self) -> usize {
311            self.0.len()
312        }
313    }
314
315    impl<'a, T: TokenTrait, TData> TokenSender<T, TData> for GeneratorTokenQueueBuffer<'a, T, TData> {
316        fn push(&mut self, token: Token<T, TData>) -> bool {
317            self.0.push_back(token);
318            true
319        }
320    }
321
322    impl<T, TData, TGeneratorState, TGenerator>
323        GeneratorTokenQueue<T, TData, TGeneratorState, TGenerator>
324    where
325        T: TokenTrait,
326        for<'a> TGenerator:
327            FnMut(&mut TGeneratorState, &mut GeneratorTokenQueueBuffer<'a, T, TData>),
328    {
329        /// Create a new [`GeneratorTokenQueue`] with a lexer function and initial state
330        pub fn new(generator: TGenerator, generator_state: TGeneratorState) -> Self {
331            GeneratorTokenQueue {
332                generator,
333                generator_state,
334                cache: VecDeque::new(),
335            }
336        }
337    }
338
339    impl<T, TData, TGeneratorState, TGenerator> TokenReader<T, TData>
340        for GeneratorTokenQueue<T, TData, TGeneratorState, TGenerator>
341    where
342        T: TokenTrait,
343        TData: Debug,
344        for<'a> TGenerator:
345            FnMut(&mut TGeneratorState, &mut GeneratorTokenQueueBuffer<'a, T, TData>),
346    {
347        fn peek(&mut self) -> Option<&Token<T, TData>> {
348            if self.cache.is_empty() {
349                (self.generator)(
350                    &mut self.generator_state,
351                    &mut GeneratorTokenQueueBuffer(&mut self.cache),
352                );
353            }
354            self.cache.front()
355        }
356
357        fn peek_n(&mut self, n: usize) -> Option<&Token<T, TData>> {
358            while self.cache.len() <= n {
359                (self.generator)(
360                    &mut self.generator_state,
361                    &mut GeneratorTokenQueueBuffer(&mut self.cache),
362                );
363            }
364            self.cache.get(n)
365        }
366
367        fn next(&mut self) -> Option<Token<T, TData>> {
368            if !self.cache.is_empty() {
369                return self.cache.pop_front();
370            }
371            (self.generator)(
372                &mut self.generator_state,
373                &mut GeneratorTokenQueueBuffer(&mut self.cache),
374            );
375            self.cache.pop_front()
376        }
377
378        fn scan(&mut self, mut cb: impl FnMut(&T, &TData) -> bool) -> Option<&Token<T, TData>> {
379            let cb = &mut cb;
380            let found = scan_cache(&mut self.cache, cb);
381            let mut return_next = match found {
382                ScanCacheResult::RetrievableInCacheAt(idx) => return self.cache.get(idx),
383                ScanCacheResult::Found => true,
384                ScanCacheResult::NotFound => false,
385            };
386            let mut found = None::<usize>;
387            while found.is_none() {
388                let start = self.cache.len();
389                (self.generator)(
390                    &mut self.generator_state,
391                    &mut GeneratorTokenQueueBuffer(&mut self.cache),
392                );
393                if self.cache.is_empty() {
394                    return None;
395                }
396                for (idx, token) in self.cache.iter().enumerate().skip(start) {
397                    if return_next {
398                        found = Some(idx);
399                        break;
400                    }
401                    if cb(&token.0, &token.1) {
402                        return_next = true;
403                    }
404                }
405            }
406            self.cache.get(found.unwrap())
407        }
408
409        fn peek_mut(&mut self) -> Option<&mut Token<T, TData>> {
410            if self.cache.is_empty() {
411                (self.generator)(
412                    &mut self.generator_state,
413                    &mut GeneratorTokenQueueBuffer(&mut self.cache),
414                );
415            }
416            self.cache.front_mut()
417        }
418    }
419}
420
421enum ScanCacheResult {
422    RetrievableInCacheAt(usize),
423    NotFound,
424    // Aka pull out next one
425    Found,
426}
427
428/// Returns the idx of the **next item** after cb returns true
429/// This returns the idx instead of the item for lifetime reasons
430fn scan_cache<T: TokenTrait, TData>(
431    cache: &mut VecDeque<Token<T, TData>>,
432    cb: &mut impl FnMut(&T, &TData) -> bool,
433) -> ScanCacheResult {
434    let mut cb_returned_true_at_idx = None::<usize>;
435    // Try to find in idx. Returns the idx when found
436    for (idx, token) in cache.iter().enumerate() {
437        if cb(&token.0, &token.1) {
438            cb_returned_true_at_idx = Some(idx);
439            break;
440        }
441    }
442    if let Some(idx) = cb_returned_true_at_idx {
443        if idx + 1 < cache.len() {
444            ScanCacheResult::RetrievableInCacheAt(idx + 1)
445        } else {
446            ScanCacheResult::Found
447        }
448    } else {
449        ScanCacheResult::NotFound
450    }
451}
452
453#[cfg(feature = "sized-tokens")]
454pub mod sized_tokens {
455    use crate::{Token, TokenReader, TokenTrait};
456
457    /// Tokens with a known length (in bytes)
458    pub trait SizedToken: TokenTrait {
459        fn length(&self) -> u32;
460    }
461
462    pub type TokenStart = source_map::Start;
463    pub type TokenEnd = source_map::End;
464
465    impl<T: SizedToken> Token<T, TokenStart> {
466        pub fn get_span(&self) -> source_map::Span {
467            let start = self.1 .0;
468            source_map::Span {
469                start,
470                end: self.1 .0 + self.0.length(),
471                source: (),
472            }
473        }
474
475        pub fn get_end(&self) -> TokenEnd {
476            source_map::End(self.1 .0 + self.0.length())
477        }
478    }
479
480    pub trait TokenReaderWithTokenEnds<T: SizedToken>: TokenReader<T, TokenStart> {
481        fn expect_next_get_end(
482            &mut self,
483            expected_type: T,
484        ) -> Result<TokenEnd, Option<(T, Token<T, TokenStart>)>> {
485            match self.next() {
486                Some(token) => {
487                    if token.0 == expected_type {
488                        Ok(token.get_end())
489                    } else if token.0.is_skippable() {
490                        // This will advance to the next, won't cyclically recurse
491                        self.expect_next_get_end(expected_type)
492                    } else {
493                        Err(Some((expected_type, token)))
494                    }
495                }
496                None => Err(None),
497            }
498        }
499    }
500
501    impl<T, TR> TokenReaderWithTokenEnds<T> for TR
502    where
503        T: SizedToken,
504        TR: TokenReader<T, TokenStart>,
505    {
506    }
507}
508
509#[cfg(test)]
510mod tests {
511    use super::*;
512
513    impl TokenTrait for u32 {}
514
515    mod buffered_token_queue {
516        use super::{BufferedTokenQueue, Token, TokenReader, TokenSender};
517
518        #[test]
519        fn next() {
520            let mut btq = BufferedTokenQueue::new();
521            btq.push(Token(12, ()));
522            btq.push(Token(32, ()));
523            btq.push(Token(52, ()));
524
525            assert_eq!(btq.next().unwrap(), Token(12, ()));
526            assert_eq!(btq.next().unwrap(), Token(32, ()));
527            assert_eq!(btq.next().unwrap(), Token(52, ()));
528            assert!(btq.next().is_none());
529        }
530
531        #[test]
532        fn peek() {
533            let mut btq = BufferedTokenQueue::new();
534            btq.push(Token(12, ()));
535
536            assert_eq!(btq.peek().unwrap(), &Token(12, ()));
537            assert_eq!(btq.next().unwrap(), Token(12, ()));
538            assert!(btq.next().is_none());
539        }
540
541        #[test]
542        fn peek_n() {
543            let mut btq = BufferedTokenQueue::new();
544            btq.push(Token(12, ()));
545            btq.push(Token(32, ()));
546            btq.push(Token(52, ()));
547
548            assert_eq!(btq.peek_n(2).unwrap(), &Token(52, ()));
549            assert_eq!(btq.next().unwrap(), Token(12, ()));
550            assert_eq!(btq.next().unwrap(), Token(32, ()));
551            assert_eq!(btq.next().unwrap(), Token(52, ()));
552            assert!(btq.next().is_none());
553        }
554
555        #[test]
556        fn expect_next() {
557            let mut btq = BufferedTokenQueue::new();
558            btq.push(Token(12, ()));
559            btq.push(Token(24, ()));
560
561            assert_eq!(btq.expect_next(12).unwrap(), ());
562            assert!(btq.expect_next(10).is_err());
563            assert!(btq.next().is_none());
564        }
565
566        #[test]
567        fn scan() {
568            let mut btq = BufferedTokenQueue::new();
569            for val in vec![4, 10, 100, 200] {
570                btq.push(Token(val, ()));
571            }
572
573            let mut count = 0;
574            let x = btq.scan(move |token_val, _| {
575                count += token_val;
576                count > 100
577            });
578            assert_eq!(x.unwrap().0, 200);
579
580            let mut count = 0;
581            let y = btq.scan(move |token_val, _| {
582                count += token_val;
583                count > 1000
584            });
585            assert_eq!(y, None);
586
587            assert_eq!(btq.next().unwrap().0, 4);
588            assert_eq!(btq.next().unwrap().0, 10);
589            assert_eq!(btq.next().unwrap().0, 100);
590            assert_eq!(btq.next().unwrap().0, 200);
591            assert!(btq.next().is_none());
592        }
593    }
594
595    mod parallel_token_queue {
596        use super::{ParallelTokenQueue, Token, TokenReader, TokenSender};
597
598        #[test]
599        fn next() {
600            let (mut sender, mut reader) = ParallelTokenQueue::new();
601            std::thread::spawn(move || {
602                sender.push(Token(12, ()));
603                sender.push(Token(32, ()));
604                sender.push(Token(52, ()));
605            });
606
607            assert_eq!(reader.next().unwrap(), Token(12, ()));
608            assert_eq!(reader.next().unwrap(), Token(32, ()));
609            assert_eq!(reader.next().unwrap(), Token(52, ()));
610            assert!(reader.next().is_none());
611        }
612
613        #[test]
614        fn peek() {
615            let (mut sender, mut reader) = ParallelTokenQueue::new();
616            std::thread::spawn(move || {
617                sender.push(Token(12, ()));
618            });
619
620            assert_eq!(reader.peek().unwrap(), &Token(12, ()));
621            assert_eq!(reader.next().unwrap(), Token(12, ()));
622            assert!(reader.next().is_none());
623        }
624
625        #[test]
626        fn next_n() {
627            let (mut sender, mut reader) = ParallelTokenQueue::new();
628            std::thread::spawn(move || {
629                sender.push(Token(12, ()));
630                sender.push(Token(32, ()));
631                sender.push(Token(52, ()));
632            });
633
634            assert_eq!(reader.peek_n(2).unwrap(), &Token(52, ()));
635            assert_eq!(reader.next().unwrap(), Token(12, ()));
636            assert_eq!(reader.next().unwrap(), Token(32, ()));
637            assert_eq!(reader.next().unwrap(), Token(52, ()));
638            assert!(reader.next().is_none());
639        }
640
641        #[test]
642        fn expect_next() {
643            let (mut sender, mut reader) = ParallelTokenQueue::new();
644            std::thread::spawn(move || {
645                sender.push(Token(12, ()));
646                sender.push(Token(24, ()));
647            });
648
649            assert_eq!(reader.expect_next(12).unwrap(), ());
650            assert!(reader.expect_next(10).is_err());
651            assert!(reader.next().is_none());
652        }
653
654        #[test]
655        fn scan() {
656            let (mut sender, mut reader) = ParallelTokenQueue::new();
657            std::thread::spawn(move || {
658                for val in vec![4, 10, 100, 200] {
659                    sender.push(Token(val, ()));
660                }
661            });
662
663            let mut count = 0;
664            let x = reader.scan(move |token_val, _| {
665                count += token_val;
666                count > 100
667            });
668            assert_eq!(x.unwrap().0, 200);
669
670            let mut count = 0;
671            let y = reader.scan(move |token_val, _| {
672                count += token_val;
673                count > 1000
674            });
675            assert_eq!(y, None);
676            assert_eq!(reader.next().unwrap().0, 4);
677            assert_eq!(reader.next().unwrap().0, 10);
678            assert_eq!(reader.next().unwrap().0, 100);
679            assert_eq!(reader.next().unwrap().0, 200);
680            assert!(reader.next().is_none());
681        }
682    }
683
684    mod generator_token_queue {
685        use super::{
686            GeneratorTokenQueue, GeneratorTokenQueueBuffer, Token, TokenReader, TokenSender,
687        };
688
689        fn lexer(state: &mut u32, sender: &mut GeneratorTokenQueueBuffer<u32, ()>) {
690            *state += 1;
691            match state {
692                1..=3 => {
693                    sender.push(Token(*state * 2, ()));
694                }
695                _ => {}
696            }
697        }
698
699        #[test]
700        fn next() {
701            let mut reader = GeneratorTokenQueue::new(lexer, 0);
702
703            assert_eq!(reader.next().unwrap(), Token(2, ()));
704            assert_eq!(reader.next().unwrap(), Token(4, ()));
705            assert_eq!(reader.next().unwrap(), Token(6, ()));
706            assert!(reader.next().is_none());
707        }
708
709        #[test]
710        fn peek() {
711            let mut reader = GeneratorTokenQueue::new(lexer, 0);
712            assert_eq!(reader.peek().unwrap(), &Token(2, ()));
713            assert_eq!(reader.next().unwrap(), Token(2, ()));
714        }
715
716        #[test]
717        fn peek_n() {
718            let mut reader = GeneratorTokenQueue::new(lexer, 0);
719
720            assert_eq!(reader.peek_n(2).unwrap(), &Token(6, ()));
721            assert_eq!(reader.next().unwrap(), Token(2, ()));
722            assert_eq!(reader.next().unwrap(), Token(4, ()));
723            assert_eq!(reader.next().unwrap(), Token(6, ()));
724            assert!(reader.next().is_none());
725        }
726
727        #[test]
728        fn expect_next() {
729            let mut reader = GeneratorTokenQueue::new(lexer, 0);
730
731            assert!(reader.expect_next(2).is_ok());
732            assert!(reader.expect_next(5).is_err());
733            assert!(reader.expect_next(6).is_ok());
734        }
735
736        #[test]
737        fn scan() {
738            let mut reader = GeneratorTokenQueue::new(lexer, 0);
739
740            let mut count = 0;
741            let x = reader.scan(move |token_val, _| {
742                count += token_val;
743                count > 3
744            });
745            assert_eq!(x.unwrap().0, 6);
746            assert_eq!(reader.next().unwrap(), Token(2, ()));
747        }
748    }
749
750    #[test]
751    fn conditional_next() {
752        let mut btq = BufferedTokenQueue::new();
753        btq.push(Token(12, ()));
754        btq.push(Token(32, ()));
755
756        assert_eq!(btq.conditional_next(|t| *t == 28), None);
757        assert_eq!(btq.conditional_next(|t| *t == 12), Some(Token(12, ())));
758        assert_eq!(btq.next().unwrap(), Token(32, ()));
759        assert!(btq.next().is_none());
760    }
761
762    mod skippable_token {
763        use super::{BufferedTokenQueue, Token, TokenReader, TokenSender, TokenTrait};
764
765        #[derive(PartialEq, Eq, Debug)]
766        enum TokenType {
767            A,
768            B,
769            Ignore,
770        }
771
772        impl TokenTrait for TokenType {
773            fn is_skippable(&self) -> bool {
774                matches!(self, TokenType::Ignore)
775            }
776        }
777
778        #[test]
779        fn still_show_with_next() {
780            let mut btq = BufferedTokenQueue::new();
781            generate_tokens(&mut btq);
782
783            assert_eq!(btq.next().unwrap(), Token(TokenType::A, ()));
784            assert_eq!(btq.next().unwrap(), Token(TokenType::Ignore, ()));
785            assert_eq!(btq.next().unwrap(), Token(TokenType::B, ()));
786            assert!(btq.next().is_none());
787        }
788
789        #[test]
790        fn skipped_under_expect_next() {
791            let mut btq = BufferedTokenQueue::new();
792            generate_tokens(&mut btq);
793
794            assert!(btq.expect_next(TokenType::A).is_ok());
795            assert!(btq.expect_next(TokenType::B).is_ok());
796            assert!(btq.next().is_none());
797        }
798
799        fn generate_tokens(btq: &mut BufferedTokenQueue<TokenType, ()>) {
800            btq.push(Token(TokenType::A, ()));
801            btq.push(Token(TokenType::Ignore, ()));
802            btq.push(Token(TokenType::B, ()));
803        }
804    }
805}
tokenizer_lib/lib.rs

tokenizer_lib/
lib.rs
