use crate::domain::{LanguageRules, PartialState};
#[cfg(test)]
mod tests;
pub enum ParsingInput<'a> {
Text(&'a str),
Chunks(Box<dyn Iterator<Item = &'a str> + 'a>),
}
pub enum ParsingOutput {
State(Box<PartialState>),
States(Vec<PartialState>),
}
pub trait ParseStrategy: Send + Sync {
fn parse(
&self,
input: ParsingInput,
rules: &dyn LanguageRules,
) -> Result<ParsingOutput, ParseError>;
fn supports_streaming(&self) -> bool;
fn optimal_chunk_size(&self) -> usize;
}
#[derive(Debug, thiserror::Error)]
pub enum ParseError {
#[error("Invalid UTF-8 in input")]
InvalidUtf8,
#[error("Empty input")]
EmptyInput,
#[error("Parsing failed: {0}")]
ParsingFailed(String),
}
pub struct SequentialParser {
chunk_size: usize,
}
impl SequentialParser {
pub fn new() -> Self {
Self {
chunk_size: 65536, }
}
pub fn with_chunk_size(chunk_size: usize) -> Self {
Self { chunk_size }
}
}
impl Default for SequentialParser {
fn default() -> Self {
Self::new()
}
}
impl ParseStrategy for SequentialParser {
fn parse(
&self,
input: ParsingInput,
rules: &dyn LanguageRules,
) -> Result<ParsingOutput, ParseError> {
match input {
ParsingInput::Text(text) => {
if text.is_empty() {
return Err(ParseError::EmptyInput);
}
let state = crate::application::parser::scan_chunk(text, rules);
Ok(ParsingOutput::State(Box::new(state)))
}
ParsingInput::Chunks(chunks) => {
let mut states = Vec::new();
for chunk in chunks {
if !chunk.is_empty() {
let state = crate::application::parser::scan_chunk(chunk, rules);
states.push(state);
}
}
if states.is_empty() {
return Err(ParseError::EmptyInput);
}
Ok(ParsingOutput::States(states))
}
}
}
fn supports_streaming(&self) -> bool {
false
}
fn optimal_chunk_size(&self) -> usize {
self.chunk_size
}
}
pub struct StreamingParser {
buffer_size: usize,
overlap_size: usize,
}
impl StreamingParser {
pub fn new() -> Self {
Self {
buffer_size: 1_048_576, overlap_size: 256, }
}
pub fn with_buffer_size(buffer_size: usize, overlap_size: usize) -> Self {
Self {
buffer_size,
overlap_size,
}
}
}
impl Default for StreamingParser {
fn default() -> Self {
Self::new()
}
}
impl ParseStrategy for StreamingParser {
fn parse(
&self,
input: ParsingInput,
rules: &dyn LanguageRules,
) -> Result<ParsingOutput, ParseError> {
match input {
ParsingInput::Text(text) => {
if text.is_empty() {
return Err(ParseError::EmptyInput);
}
let state = crate::application::parser::scan_chunk(text, rules);
Ok(ParsingOutput::State(Box::new(state)))
}
ParsingInput::Chunks(chunks) => {
let mut states = Vec::new();
let mut overlap = String::new();
for chunk in chunks {
let combined = if overlap.is_empty() {
chunk.to_string()
} else {
format!("{overlap}{chunk}")
};
if !combined.is_empty() {
let state = crate::application::parser::scan_chunk(&combined, rules);
states.push(state);
if chunk.len() >= self.overlap_size {
overlap = chunk[chunk.len() - self.overlap_size..].to_string();
} else {
overlap = chunk.to_string();
}
}
}
if states.is_empty() {
return Err(ParseError::EmptyInput);
}
Ok(ParsingOutput::States(states))
}
}
}
fn supports_streaming(&self) -> bool {
true
}
fn optimal_chunk_size(&self) -> usize {
self.buffer_size
}
}