mod sentence;
mod tokenizer;
pub use sentence::{Sentence, SentenceSplitter};
pub use tokenizer::{CodeTokenStream, CodeTokenizer, TokenType, TypedToken};
#[derive(Debug, Clone)]
pub struct ProcessedSentence {
pub text: String,
pub tokens: Vec<String>,
pub is_code: bool,
pub offset: usize,
}
#[derive(Clone)]
pub struct TextProcessor {
sentence_splitter: SentenceSplitter,
tokenizer: CodeTokenizer,
}
impl Default for TextProcessor {
fn default() -> Self {
Self::new()
}
}
impl TextProcessor {
pub fn new() -> Self {
Self {
sentence_splitter: SentenceSplitter::new(),
tokenizer: CodeTokenizer::new(),
}
}
pub fn process_chunk(&self, text: &str) -> Vec<ProcessedSentence> {
let sentences = self.sentence_splitter.split(text);
sentences
.into_iter()
.map(|sentence| {
let tokens = self.tokenizer.tokenize(&sentence.text);
ProcessedSentence {
text: sentence.text,
tokens,
is_code: sentence.is_code,
offset: sentence.offset,
}
})
.collect()
}
pub fn tokenizer(&self) -> &CodeTokenizer {
&self.tokenizer
}
pub fn sentence_splitter(&self) -> &SentenceSplitter {
&self.sentence_splitter
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_text_processor_basic() {
let processor = TextProcessor::new();
let text = "This function parses JSON data.";
let processed = processor.process_chunk(text);
assert_eq!(processed.len(), 1);
assert!(!processed[0].is_code);
assert!(processed[0].tokens.contains(&"pars".to_string())); assert!(processed[0].tokens.contains(&"JSON".to_string())); }
#[test]
fn test_text_processor_code() {
let processor = TextProcessor::new();
let code = "fn parseJSONData(input: &str) -> Result<Data, Error> {\n todo!()\n}";
let processed = processor.process_chunk(code);
assert!(!processed.is_empty());
assert!(processed[0].is_code);
}
#[test]
fn test_text_processor_mixed() {
let processor = TextProcessor::new();
let mixed = "This is a description.\nfn example() { }\nAnother line.";
let processed = processor.process_chunk(mixed);
let has_code = processed.iter().any(|p| p.is_code);
let has_prose = processed.iter().any(|p| !p.is_code);
assert!(has_code);
assert!(has_prose);
}
#[test]
fn test_text_processor_empty() {
let processor = TextProcessor::new();
let processed = processor.process_chunk("");
assert!(processed.is_empty());
}
#[test]
fn test_tokens_contain_split_identifiers() {
let processor = TextProcessor::new();
let text = "The getUserById function returns a User.";
let processed = processor.process_chunk(text);
assert_eq!(processed.len(), 1);
let tokens = &processed[0].tokens;
assert!(tokens.contains(&"get".to_string()));
assert!(tokens.contains(&"user".to_string()));
assert!(tokens.contains(&"by".to_string()));
assert!(tokens.contains(&"id".to_string()));
}
}