use crate::core::analysis::analyzer::{Analyzer, AnalyzerStoredValue, TokenStreamComponents};
use crate::core::analysis::char_array_set::CharArraySet;
use crate::core::analysis::filtering_token_filter::FilteringTokenFilter;
use crate::core::analysis::lower_case_filter::LowerCaseFilter;
use crate::core::analysis::standard::standard_tokenizer::{
MAX_TOKEN_LENGTH_LIMIT, StandardTokenizer,
};
use crate::core::analysis::stop_filter::StopFilter;
use crate::core::analysis::stop_word_analyzer_base::{StopWordAnalyzerBase, init_stop_wors};
use crate::core::analysis::token_stream::NormalizeTokenStream;
use crate::core::util::error::lucene_error::LuceneError;
use crate::core::util::error::lucene_error::Result;
use std::sync::LazyLock;
static EMPTY_STOP_WORDS: LazyLock<CharArraySet> = LazyLock::new(|| init_stop_wors(None));
pub const DEFAULT_MAX_TOKEN_LENGTH: usize = 255;
pub struct StandardAnalyzer {
max_token_length: usize,
stored_value: AnalyzerStoredValue,
}
impl Default for StandardAnalyzer {
fn default() -> Self {
Self::new()
}
}
impl StandardAnalyzer {
pub fn new() -> Self {
Self {
max_token_length: DEFAULT_MAX_TOKEN_LENGTH,
stored_value: AnalyzerStoredValue::new(),
}
}
pub fn set_max_token_length(&mut self, length: usize) -> Result<()> {
if length < 1 {
return Err(LuceneError::illegal_argument(
"maxTokenLength must be greater than zero",
));
} else if length > MAX_TOKEN_LENGTH_LIMIT {
return Err(LuceneError::illegal_argument(format!(
"maxTokenLength may not exceed {MAX_TOKEN_LENGTH_LIMIT}"
)));
}
self.max_token_length = length;
Ok(())
}
pub fn get_max_token_length(&self) -> usize {
self.max_token_length
}
}
pub type StandardAnalyzerTS = FilteringTokenFilter<LowerCaseFilter<StandardTokenizer>, StopFilter>;
impl Analyzer for StandardAnalyzer {
fn create_components(&self, _field: &str) -> Result<TokenStreamComponents> {
let mut src = StandardTokenizer::new()?;
src.set_max_token_length(self.max_token_length)?;
let tok = StopFilter::with_empty_stop_words(LowerCaseFilter::new(src));
Ok(TokenStreamComponents::new(tok, Some(self.max_token_length)))
}
fn stored_value(&self) -> &AnalyzerStoredValue {
&self.stored_value
}
fn normalize_from_ts(
&self,
_field_name: &str,
in_: NormalizeTokenStream,
) -> Result<NormalizeTokenStream> {
Ok(LowerCaseFilter::new(Box::new(in_)).into())
}
}
crate::impl_analyzer_close!(StandardAnalyzer);
impl StopWordAnalyzerBase for StandardAnalyzer {
fn get_stop_words(&self) -> &CharArraySet {
&EMPTY_STOP_WORDS
}
}