tokmat 0.3.3

//! Configurable definition of a "word" -- the character class that delimits tokens.
//!
//! The tokenizer, TEL compiler, and extractor all share one notion of which
//! characters make up a word. Historically that was the hardcoded `[\w\-']`
//! repeated as `const`s across three modules. It is really *metadata*: a model
//! may redefine the word character set, so it must be changeable without
//! recompiling.
//!
//! This module is the single source of truth. All three regex forms a caller
//! needs -- the bare character class, the one-or-more word match, and the
//! zero-width word boundary -- are *derived* from one character-class spec, so
//! they can never drift apart. A [`crate::token_model::TokenModel`] retains its
//! own `WordDefinition` for deterministic per-model tokenization, and the TEL
//! compiler / extractor thread a `WordDefinition` explicitly through
//! [`crate::tel::CompiledPattern::compile_with_word_definition`] and
//! [`crate::extractor::Extractor::with_word_definition`] so concurrent
//! extraction of *different* models never shares mutable state.
//!
//! The process-wide definition (defaulting to the wanParser-compatible `\w\-'`,
//! configurable via [`configure_word_definition`]) remains only as the fallback
//! used by the convenience [`crate::tel::CompiledPattern::compile`] and legacy
//! global-tokenizer paths. It is *not* read on any per-model extraction path,
//! so it can no longer be a concurrency hazard there.

use pcre2::bytes::{Regex as Pcre2Regex, RegexBuilder as Pcre2RegexBuilder};
use serde::{Deserialize, Serialize};
use std::sync::{Arc, LazyLock, RwLock};

/// Default word character-class members: word chars, hyphen, apostrophe.
///
/// This is the inner of the character class (no surrounding brackets).
pub const DEFAULT_WORD_CHARS: &str = r"\w\-'";

/// A word definition: the set of characters that constitute a word.
///
/// Construct from the character-class contents (without brackets), e.g.
/// `WordDefinition::new(r"\w\-'")`. The regex forms used across the crate are
/// derived from it so they stay consistent.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct WordDefinition {
    chars: String,
}

impl WordDefinition {
    /// Build a definition from the character-class contents (no brackets).
    ///
    /// Surrounding `[` `]` are tolerated and stripped so both `\w\-'` and
    /// `[\w\-']` are accepted.
    #[must_use]
    pub fn new(chars: impl Into<String>) -> Self {
        let chars = chars.into();
        let trimmed = chars.trim();
        let inner = trimmed
            .strip_prefix('[')
            .and_then(|value| value.strip_suffix(']'))
            .unwrap_or(trimmed);
        Self {
            chars: inner.to_string(),
        }
    }

    /// The character-class contents, e.g. `\w\-'` (no brackets).
    #[must_use]
    pub fn chars(&self) -> &str {
        &self.chars
    }

    /// The bracketed character class, e.g. `[\w\-']`.
    #[must_use]
    pub fn char_class(&self) -> String {
        format!("[{}]", self.chars)
    }

    /// A one-or-more word match, e.g. `[\w\-']+`.
    #[must_use]
    pub fn word_regex(&self) -> String {
        format!("[{}]+", self.chars)
    }

    /// A zero-width word boundary built from this definition.
    #[must_use]
    pub fn boundary(&self) -> String {
        let class = self.char_class();
        format!("(?:(?={class})(?<!{class})|(?<={class})(?!{class}))")
    }

    /// Compile this definition's zero-width word boundary into a PCRE2 regex.
    ///
    /// Lets a [`crate::token_model::TokenModel`] hold its own compiled boundary so
    /// the per-model tokenize path does not depend on the process-global state.
    ///
    /// # Panics
    ///
    /// Panics if the derived boundary regex is not a valid PCRE2 pattern.
    #[must_use]
    pub fn compiled_boundary(&self) -> Pcre2Regex {
        compile_boundary(&self.boundary())
    }
}

impl Default for WordDefinition {
    fn default() -> Self {
        Self::new(DEFAULT_WORD_CHARS)
    }
}

/// The active definition, its compiled tokenizer boundary regex, and the
/// pre-rendered derived string forms.
///
/// The boundary regex *and* the derived regex-fragment strings are built once
/// when the definition is set, so hot paths never pay a PCRE2 compile or a
/// `format!`/`String` allocation per row -- they only clone a cheap `Arc`.
struct WordState {
    definition: WordDefinition,
    boundary: Arc<Pcre2Regex>,
    char_class: Arc<str>,
    word_regex: Arc<str>,
    boundary_str: Arc<str>,
}

impl WordState {
    fn build(definition: WordDefinition) -> Self {
        let boundary_str: Arc<str> = Arc::from(definition.boundary());
        let boundary = Arc::new(compile_boundary(&boundary_str));
        Self {
            char_class: Arc::from(definition.char_class()),
            word_regex: Arc::from(definition.word_regex()),
            boundary_str,
            definition,
            boundary,
        }
    }
}

static STATE: LazyLock<RwLock<WordState>> =
    LazyLock::new(|| RwLock::new(WordState::build(WordDefinition::default())));

/// Configure the process-wide word definition (e.g. from model metadata).
///
/// Recompiles the cached tokenizer boundary regex. Intended to be called once at
/// model-load time; until then the default (`\w\-'`) is used.
///
/// # Panics
///
/// Panics if the derived boundary regex is not a valid PCRE2 pattern.
pub fn configure_word_definition(definition: WordDefinition) {
    let next = WordState::build(definition);
    *STATE.write().expect("word definition lock poisoned") = next;
}

/// The active word definition (the configured one, or the default).
#[must_use]
pub fn word_definition() -> WordDefinition {
    STATE
        .read()
        .expect("word definition lock poisoned")
        .definition
        .clone()
}

/// The compiled tokenizer word-boundary regex for the active definition.
///
/// Cached, so the tokenizer hot path pays only a lock read and an `Arc` clone --
/// not a PCRE2 compile -- per call.
#[must_use]
pub fn tokenizer_boundary() -> Arc<Pcre2Regex> {
    STATE
        .read()
        .expect("word definition lock poisoned")
        .boundary
        .clone()
}

/// The bracketed word character class (e.g. `[\w\-']`) for the active definition.
///
/// Pre-rendered and cached: callers pay only a lock read and an `Arc` clone, not
/// a `String` allocation, per call.
#[must_use]
pub fn char_class() -> Arc<str> {
    STATE
        .read()
        .expect("word definition lock poisoned")
        .char_class
        .clone()
}

/// The one-or-more word match (e.g. `[\w\-']+`) for the active definition.
///
/// Pre-rendered and cached (see [`char_class`]).
#[must_use]
pub fn word_regex() -> Arc<str> {
    STATE
        .read()
        .expect("word definition lock poisoned")
        .word_regex
        .clone()
}

/// The zero-width word-boundary pattern string for the active definition.
///
/// Pre-rendered and cached (see [`char_class`]). This is the regex *source*; the
/// compiled tokenizer boundary is [`tokenizer_boundary`].
#[must_use]
pub fn boundary_pattern() -> Arc<str> {
    STATE
        .read()
        .expect("word definition lock poisoned")
        .boundary_str
        .clone()
}

fn compile_boundary(pattern: &str) -> Pcre2Regex {
    Pcre2RegexBuilder::new()
        .utf(true)
        .ucp(true)
        .jit_if_available(true)
        .build(pattern)
        .expect("valid word boundary regex")
}

/// Serializes tests that mutate or depend on the process-global word-definition
/// [`STATE`] (mutated only by [`configure_word_definition`]). Under the parallel
/// test runner a test that temporarily reconfigures the definition would
/// otherwise race tests that read the default boundary. Any such test must hold
/// this lock for its duration.
#[cfg(test)]
pub(crate) static WORD_DEF_TEST_LOCK: RwLock<()> = RwLock::new(());

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn default_matches_legacy_hardcoded_forms() {
        let def = WordDefinition::default();
        assert_eq!(def.char_class(), r"[\w\-']");
        assert_eq!(def.word_regex(), r"[\w\-']+");
        assert_eq!(
            def.boundary(),
            r"(?:(?=[\w\-'])(?<![\w\-'])|(?<=[\w\-'])(?![\w\-']))"
        );
    }

    #[test]
    fn new_strips_surrounding_brackets() {
        assert_eq!(WordDefinition::new(r"[\w\-']").chars(), r"\w\-'");
        assert_eq!(WordDefinition::new(r"\w\-'").chars(), r"\w\-'");
    }

    #[test]
    fn configured_definition_changes_derived_forms_and_is_reset() {
        // Exclusive: this test mutates the process-global definition.
        let _guard = WORD_DEF_TEST_LOCK
            .write()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        configure_word_definition(WordDefinition::new(r"\w"));
        assert_eq!(word_definition().word_regex(), r"[\w]+");
        // restore the default so other tests see legacy behaviour
        configure_word_definition(WordDefinition::default());
        assert_eq!(word_definition().word_regex(), r"[\w\-']+");
    }
}