use std::{
collections::VecDeque,
cell::UnsafeCell,
fmt,
sync::mpsc::{sync_channel, Receiver, RecvError, SyncSender},
};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Span(pub usize, pub usize);
#[derive(PartialEq, Eq)]
pub struct Token<T>(pub T, pub Span);
impl<T: fmt::Debug> fmt::Debug for Token<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?} {}:{}", self.0, self.1 .0, self.1 .1)
}
}
pub struct ParseError {
pub reason: String,
pub position: Option<Span>,
}
impl fmt::Debug for ParseError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"ParseError: {}{}",
self.reason,
if let Some(pos) = &self.position {
format!(" {:?}", pos)
} else {
"".to_owned()
}
)
}
}
pub trait TokenReader<T>
where
T: PartialEq + fmt::Debug,
{
fn peek(&self) -> Option<&Token<T>>;
fn next(&mut self) -> Option<Token<T>>;
fn scan(&self, f: impl FnMut(&T) -> bool) -> Option<&Token<T>>;
fn expect_next(&mut self, expected_type: T) -> Result<Span, ParseError> {
match self.next() {
Some(Token(token_type, position)) => {
if token_type != expected_type {
Err(ParseError {
reason: format!("Expected {:?}, received {:?}", expected_type, token_type),
position: Some(position),
})
} else {
Ok(position)
}
}
None => Err(ParseError {
reason: format!("Expected {:?} but reached end of source", expected_type),
position: None,
}),
}
}
}
pub trait TokenSender<T> {
fn push(&mut self, token: Token<T>);
}
pub struct StaticTokenChannel<T> {
tokens: VecDeque<Token<T>>,
}
impl<T> StaticTokenChannel<T> {
pub fn new() -> Self {
StaticTokenChannel {
tokens: VecDeque::new(),
}
}
}
impl<T> TokenSender<T> for StaticTokenChannel<T> {
fn push(&mut self, token: Token<T>) {
self.tokens.push_back(token)
}
}
impl<T: PartialEq + fmt::Debug> TokenReader<T> for StaticTokenChannel<T> {
fn peek(&self) -> Option<&Token<T>> {
self.tokens.front()
}
fn next(&mut self) -> Option<Token<T>> {
self.tokens.pop_front()
}
fn scan(&self, mut cb: impl FnMut(&T) -> bool) -> Option<&Token<T>> {
let mut iter = self.tokens.iter().peekable();
while let Some(token) = iter.next() {
if cb(&token.0) {
return iter.peek().map(|v| *v);
}
}
None
}
}
pub struct StreamedTokenSender<T>(SyncSender<Token<T>>);
pub struct StreamedTokenReader<T> {
receiver: Receiver<Token<T>>,
cache: UnsafeCell<VecDeque<Token<T>>>,
}
impl<T> TokenSender<T> for StreamedTokenSender<T> {
fn push(&mut self, token: Token<T>) {
self.0.send(token).unwrap();
}
}
pub fn get_streamed_token_channel<T>() -> (StreamedTokenSender<T>, StreamedTokenReader<T>) {
let (sender, receiver) = sync_channel::<Token<T>>(20);
(
StreamedTokenSender(sender),
StreamedTokenReader {
receiver,
cache: UnsafeCell::new(VecDeque::new()),
},
)
}
impl<T: PartialEq + fmt::Debug> TokenReader<T> for StreamedTokenReader<T> {
fn peek(&self) -> Option<&Token<T>> {
let cache = unsafe { &mut *self.cache.get() };
if cache.is_empty() {
match self.receiver.recv() {
Ok(val) => cache.push_back(val),
Err(RecvError) => {
return None;
}
}
}
cache.front()
}
fn next(&mut self) -> Option<Token<T>> {
let cache = unsafe { &mut *self.cache.get() };
if !cache.is_empty() {
return cache.pop_front();
}
self.receiver.recv().ok()
}
fn scan(&self, mut cb: impl FnMut(&T) -> bool) -> Option<&Token<T>> {
for token in unsafe { &*self.cache.get() }.iter() {
if cb(&token.0) {
return Some(token);
}
}
let cache = unsafe { &mut *self.cache.get() };
let mut found = false;
loop {
match self.receiver.recv() {
Ok(val) => {
if found {
cache.push_back(val);
return cache.back();
}
if cb(&val.0) {
found = true;
}
cache.push_back(val);
}
Err(RecvError) => {
return None;
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn static_token_channel() {
let mut stc = StaticTokenChannel::new();
stc.push(Token(12, Span(0, 2)));
stc.push(Token(32, Span(2, 4)));
stc.push(Token(52, Span(4, 8)));
assert_eq!(stc.next().unwrap(), Token(12, Span(0, 2)));
assert_eq!(stc.next().unwrap(), Token(32, Span(2, 4)));
assert_eq!(stc.next().unwrap(), Token(52, Span(4, 8)));
assert!(stc.next().is_none());
}
#[test]
fn static_token_channel_peek() {
let mut stc = StaticTokenChannel::new();
stc.push(Token(12, Span(0, 2)));
assert_eq!(stc.peek().unwrap(), &Token(12, Span(0, 2)));
assert_eq!(stc.next().unwrap(), Token(12, Span(0, 2)));
assert!(stc.next().is_none());
}
#[test]
fn static_token_channel_expect_next() {
let mut stc = StaticTokenChannel::new();
stc.push(Token(12, Span(0, 2)));
stc.push(Token(24, Span(2, 4)));
assert_eq!(stc.expect_next(12).unwrap(), Span(0, 2));
let err = stc.expect_next(10).unwrap_err();
assert_eq!(err.position, Some(Span(2, 4)));
assert_eq!(err.reason, "Expected 10, received 24".to_owned());
assert!(stc.next().is_none());
}
#[test]
fn static_token_channel_scan() {
let mut stc = StaticTokenChannel::new();
for val in vec![4, 10, 100, 200] {
stc.push(Token(val, Span(0, 0)));
}
let mut count = 0;
let x = stc.scan(move |token_val| {
count += token_val;
count > 100
});
assert_eq!(x.unwrap().0, 200);
assert_eq!(stc.next().unwrap().0, 4);
assert_eq!(stc.next().unwrap().0, 10);
assert_eq!(stc.next().unwrap().0, 100);
assert_eq!(stc.next().unwrap().0, 200);
assert!(stc.next().is_none());
}
#[test]
fn streamed_token_channel() {
let (mut sender, mut reader) = get_streamed_token_channel();
std::thread::spawn(move || {
sender.push(Token(12, Span(0, 2)));
sender.push(Token(32, Span(2, 4)));
sender.push(Token(52, Span(4, 8)));
});
assert_eq!(reader.next().unwrap(), Token(12, Span(0, 2)));
assert_eq!(reader.next().unwrap(), Token(32, Span(2, 4)));
assert_eq!(reader.next().unwrap(), Token(52, Span(4, 8)));
assert!(reader.next().is_none());
}
#[test]
fn streamed_token_channel_peek() {
let (mut sender, mut reader) = get_streamed_token_channel();
std::thread::spawn(move || {
sender.push(Token(12, Span(0, 2)));
});
assert_eq!(reader.peek().unwrap(), &Token(12, Span(0, 2)));
assert_eq!(reader.next().unwrap(), Token(12, Span(0, 2)));
assert!(reader.next().is_none());
}
#[test]
fn streamed_token_channel_expect_next() {
let (mut sender, mut reader) = get_streamed_token_channel();
std::thread::spawn(move || {
sender.push(Token(12, Span(0, 2)));
sender.push(Token(24, Span(2, 4)));
});
assert_eq!(reader.expect_next(12).unwrap(), Span(0, 2));
let err = reader.expect_next(10).unwrap_err();
assert_eq!(err.position, Some(Span(2, 4)));
assert_eq!(err.reason, "Expected 10, received 24".to_owned());
assert!(reader.next().is_none());
}
#[test]
fn streamed_token_channel_scan() {
let (mut sender, mut reader) = get_streamed_token_channel();
std::thread::spawn(move || {
for val in vec![4, 10, 100, 200] {
sender.push(Token(val, Span(0, 0)));
}
});
let mut count = 0;
let x = reader.scan(move |token_val| {
count += token_val;
count > 100
});
assert_eq!(x.unwrap().0, 200);
assert_eq!(reader.next().unwrap().0, 4);
assert_eq!(reader.next().unwrap().0, 10);
assert_eq!(reader.next().unwrap().0, 100);
assert_eq!(reader.next().unwrap().0, 200);
assert!(reader.next().is_none());
}
}