//! Contains tools and structs for processing text to be inserted into the database

use serde::Deserialize;

/// Different methods for generating tokens used for full text search
#[derive(Deserialize, Debug, Eq, PartialEq)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Tokenizer {
    #[serde(rename_all = "camelCase")]
    Ngram {
        token_length: usize,
    },
    Standard,
}

impl Tokenizer {
    /// Process text and return the tokens based on the specific tokenizer
    pub fn process(&self, text: Vec<String>) -> Vec<String> {
        match self {
            Tokenizer::Ngram { token_length } => text
                .into_iter()
                .flat_map(|t| {
                    let chars = t.chars().collect::<Vec<_>>();

                    if chars.len() < *token_length {
                        return vec![];
                    }

                    let mut grams: Vec<String> = vec![];

                    for i in 0..=(chars.len() - token_length) {
                        grams.push(chars[i..i + token_length].iter().collect());
                    }

                    grams
                })
                .collect(),
            Tokenizer::Standard => text
                .iter()
                .flat_map(|t| t.split(&[' ', ',', ';', ':', '!']).map(|x| x.into()))
                .collect(),
        }
    }
}

/// Different methods for transforming text or tokens
#[derive(Deserialize, Debug, Eq, PartialEq)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum TokenFilter {
    Upcase,
    Downcase,
}

impl TokenFilter {
    /// Process text based on a specific token filter
    pub fn process(&self, text: Vec<String>) -> Vec<String> {
        match self {
            TokenFilter::Upcase => text.into_iter().map(|text| text.to_uppercase()).collect(),
            TokenFilter::Downcase => text.into_iter().map(|text| text.to_lowercase()).collect(),
        }
    }
}

#[derive(Deserialize, Debug, Eq, PartialEq)]
#[serde(untagged)]
pub enum TokenProcessor {
    TokenFilter(TokenFilter),
    Tokenizer(Tokenizer),
}

impl TokenProcessor {
    /// Process text based on a specific token filter or tokenizer and return the tokens
    pub fn process(&self, text: Vec<String>) -> Vec<String> {
        match self {
            Self::TokenFilter(filter) => filter.process(text),
            Self::Tokenizer(tokenizer) => tokenizer.process(text),
        }
    }
}

/// The combination of a tokenizer and zero or more token filters.
///
/// It is used on match, dynamic-match and field-dynamic-match indexes to prepare inserted text.
#[derive(Deserialize, Debug, Eq, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct TextPipeline {
    pub tokenizer: Tokenizer,
    pub token_filters: Vec<TokenProcessor>,
}

impl TextPipeline {
    /// Process text based on the tokenizer and token filters
    pub fn process(&self, text: Vec<String>) -> Vec<String> {
        let tokenized = self.tokenizer.process(text);
        self.token_filters
            .iter()
            .fold(tokenized, |text, pass| pass.process(text))
    }
}

#[cfg(test)]
mod tests {
    use super::{TextPipeline, TokenFilter, TokenProcessor, Tokenizer};

    #[test]
    fn test_standard() {
        let output = Tokenizer::Standard.process(vec!["Hello from Ada Lovelace".into()]);

        assert_eq!(output, vec!["Hello", "from", "Ada", "Lovelace"]);
    }

    #[test]
    fn test_ngram() {
        let output = Tokenizer::Ngram { token_length: 3 }.process(vec!["Lovelace".into()]);
        assert_eq!(output, vec!["Lov", "ove", "vel", "ela", "lac", "ace"]);
    }

    #[test]
    fn test_ngram_equal_length() {
        let output = Tokenizer::Ngram { token_length: 4 }.process(vec!["Love".into()]);
        assert_eq!(output, vec!["Love"]);
    }

    #[test]
    fn test_ngram_shorter_length() {
        let output = Tokenizer::Ngram { token_length: 4 }.process(vec!["Lov".into()]);
        assert_eq!(output, Vec::<String>::new());
    }

    #[test]
    fn test_ngram_zero_length() {
        let output = Tokenizer::Ngram { token_length: 0 }.process(vec!["Lovelace".into()]);
        assert_eq!(output, vec!["", "", "", "", "", "", "", "", ""]);
    }

    #[test]
    fn test_downcase() {
        let output = TokenFilter::Downcase.process(vec!["HeLLOWorlD".into()]);
        assert_eq!(output, vec!["helloworld"]);
    }

    #[test]
    fn test_upcase() {
        let output = TokenFilter::Upcase.process(vec!["HeLLOWorlD".into()]);
        assert_eq!(output, vec!["HELLOWORLD"]);
    }

    #[test]
    fn test_chaining_filter_and_tokenizer() {
        let pipeline = TextPipeline {
            tokenizer: Tokenizer::Ngram { token_length: 3 },
            token_filters: vec![TokenProcessor::TokenFilter(TokenFilter::Upcase)],
        };

        let output = pipeline.process(vec!["HeLlOwOrLd".into()]);

        assert_eq!(
            output,
            vec!["HEL", "ELL", "LLO", "LOW", "OWO", "WOR", "ORL", "RLD"]
        );
    }

    #[test]
    fn test_chain_tokenizers() {
        let pipeline = TextPipeline {
            tokenizer: Tokenizer::Standard,
            token_filters: vec![TokenProcessor::Tokenizer(Tokenizer::Ngram {
                token_length: 3,
            })],
        };

        let output = pipeline.process(vec!["CipherStash Dev Team".into()]);

        assert_eq!(
            output,
            vec![
                "Cip", "iph", "phe", "her", "erS", "rSt", "Sta", "tas", "ash", "Dev", "Tea", "eam"
            ]
        );
    }

    #[test]
    fn test_multiple_codepoint_chars() {
        let pipeline = TextPipeline {
            tokenizer: Tokenizer::Standard,
            token_filters: vec![TokenProcessor::Tokenizer(Tokenizer::Ngram {
                token_length: 3,
            })],
        };

        let output = pipeline.process(vec!["mañana".into()]);

        assert_eq!(output, vec!["mañ", "aña", "ñan", "ana"]);
    }
}