infino 0.2.0

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Infino Authors

//! Tokenization, plus BM25 query parsing ([`Tokenizer::parse`]).
//! The parser lives here because the `+` / `-` clause sigils must be
//! handled before tokenizing — the tokenizer splits on both.
//!
//! Ships two tokenizers: [`AsciiLowerTokenizer`] (the default) and
//! [`StandardTokenizer`]. The [`Tokenizer`] trait is the extension
//! point for ICU / language-aware stemmers / custom char filters
//! under the same trait without touching FTS code.
//!
//! [`AsciiLowerTokenizer`] semantics:
//!   - Split on any byte that isn't `[A-Za-z0-9]`.
//!   - Lowercase each ASCII letter (bytes `b'A'..=b'Z'` → `b'a'..=b'z'`).
//!   - Drop any token that contains a non-ASCII byte (high-bit set).
//!     Non-ASCII tokens are silently dropped (not an error) — the
//!     ASCII-only design is intentional; richer tokenizers can opt
//!     into the trait without changing the FTS pipeline.
//!   - Empty tokens are never emitted.
//!
//! [`StandardTokenizer`] semantics (Unicode-aware):
//!   - Segment on Unicode text boundaries (UAX #29 word boundaries),
//!     keeping runs that contain alphanumerics and discarding
//!     whitespace/punctuation-only segments.
//!   - Lowercase each token via full Unicode case folding.
//!   - Non-ASCII letters and digits are preserved (not dropped), so
//!     accented and non-Latin scripts remain searchable.
//!   - **No Unicode normalization** (NFC/NFD): a token is emitted in its
//!     input code-point encoding. This matches the `standard` analyzer in
//!     Lucene / Elasticsearch, which applies normalization only through a
//!     separate, opt-in ICU filter — never in the standard pipeline.
//!     Canonicalizing equivalent encodings (e.g. precomposed `é` vs. the
//!     base `e` + combining acute) is therefore a distinct analyzer's job:
//!     a normalizing analyzer plugs in through the [`Tokenizer`] trait
//!     rather than altering `standard`'s semantics.

use std::{
    any::Any,
    borrow::Cow,
    collections::BTreeSet,
    str::{from_utf8, from_utf8_unchecked},
    sync::Arc,
};

use unicode_segmentation::UnicodeSegmentation;
use wide::u8x16;

use super::reader::BoolMode;

/// Smallest byte value that is non-ASCII (has the high bit set). The
/// v1 ASCII-only rule drops any token containing a byte `>= this`.
const NON_ASCII_BYTE_MIN: u8 = 0x80;

/// Low-16-bit mask applied to a `u8x16` comparison bitmask, keeping
/// one bit per SIMD lane (the scan processes 16 bytes per chunk).
const LANE_BITMASK: u32 = 0xFFFF;

/// Initial capacity of the lowercase-token scratch buffer. Sized to
/// the common case of short tokens so the hot path rarely reallocs.
const TOKEN_SCRATCH_INITIAL_CAP: usize = 32;

/// Trait every tokenizer impl must satisfy.
///
/// Three entry points:
///
///   - [`Tokenizer::tokenize`] — iterator-shaped, yields owned
///     `String`s. Convenient for query-side / one-off use, but
///     allocates one heap `String` per token.
///
///   - [`Tokenizer::tokenize_each`] — callback-shaped, hands the
///     callback a `&str` borrowed from an internal scratch buffer
///     (valid only for the duration of the call). Zero-alloc on the
///     hot ingest path. The default impl wraps `tokenize`; impls
///     that can do better (like [`AsciiLowerTokenizer`]) override.
///     The callback is `&mut dyn FnMut`, so each per-token call
///     pays one indirect dispatch and LLVM cannot inline the
///     callback body into the tokenizer scan loop.
///
///   - [`Tokenizer::as_any`] — downcast hatch so the FTS build path
///     can take a monomorphic fast path when the tokenizer is the
///     default [`AsciiLowerTokenizer`]. The fast path bypasses the
///     `&mut dyn FnMut(&str)` indirection by calling the inherent
///     [`AsciiLowerTokenizer::tokenize_each_inline`] method, whose
///     `F: FnMut(&str)` parameter lets LLVM inline the callback
///     body straight into the tokenizer's per-byte scan. Custom
///     tokenizers don't need to opt in — they just return `self`
///     and never get downcast.
pub trait Tokenizer: Send + Sync + std::fmt::Debug + 'static {
    /// Stable registry name recorded in a column's stored FTS config
    /// (the `tokenizer` field of `inf.fts.columns`) and used to
    /// reconstruct the matching tokenizer at read time via
    /// [`tokenizer_for_name`]. Must round-trip:
    /// `tokenizer_for_name(t.name())` yields a tokenizer of the same
    /// kind as `t`.
    fn name(&self) -> &'static str;

    /// Yield each token as an owned `String` lower-cased per the
    /// implementation's rules.
    ///
    /// ## Phrase positions and dropped tokens
    ///
    /// The positional index numbers tokens by the order this trait
    /// yields them, and exact-phrase matching checks those numbers for
    /// adjacency. A tokenizer that *drops* an input token (a stopword
    /// filter, a non-ASCII skip, etc.) without otherwise signalling it
    /// makes the tokens on either side of the dropped one look
    /// adjacent — so a phrase can match text that isn't actually
    /// contiguous. The built-in [`AsciiLowerTokenizer`] avoids this on
    /// its positional build path by leaving a position gap for each
    /// dropped run; a custom tokenizer that drops tokens and needs
    /// exact phrase semantics must not rely on this trait to preserve
    /// gaps (position increments through the trait are a planned
    /// extension, not yet available).
    fn tokenize<'a>(&'a self, text: &'a str) -> Box<dyn Iterator<Item = String> + 'a>;

    /// Call `f(&token)` for each token. The `&str` passed to `f` is
    /// valid only for that call — copy it (e.g. into a bump arena) if
    /// you need to keep it.
    ///
    /// Default impl iterates `self.tokenize(...)` and calls `f` on
    /// each `String` (one heap alloc per token). Impls that can be
    /// zero-alloc should override.
    fn tokenize_each(&self, text: &str, f: &mut dyn FnMut(&str)) {
        for s in self.tokenize(text) {
            f(&s);
        }
    }

    /// Downcast hatch for the FTS build hot path. Default impl
    /// returns `self` cast to `&dyn Any`; concrete impls should
    /// not override unless they wrap another tokenizer.
    fn as_any(&self) -> &dyn Any;

    /// Used to tokenize a query. The tokens handed to `f` stay alive
    /// as long as `text` (the query) is alive.
    fn tokenize_each_query<'q>(&self, text: &'q str, f: &mut dyn FnMut(Cow<'q, str>)) {
        self.tokenize_each(text, &mut |t| f(Cow::Owned(t.to_owned())));
    }

    /// Used to parse a query into its clauses by leading sigil:
    /// `"+rust async -python"` → musts `["rust"]`, positives
    /// `["async"]`, negatives `["python"]`. A `+`-prefixed run is a
    /// **must** clause (the doc must contain it), a `-`-prefixed run
    /// a **must-not** clause (hard exclusion), and a bare run lands
    /// in `positives`, whose polarity the query layer resolves from
    /// the default operator (`BoolMode`). A query with no must or
    /// positive clause is not an error here; the caller checks.
    fn parse<'q>(&self, query: &'q str) -> ParsedQuery<'q> {
        let mut parsed = ParsedQuery::default();
        let bytes = query.as_bytes();
        let mut i = 0usize;
        let mut seg_start = 0usize;
        while i < bytes.len() {
            if bytes[i] != b'"' {
                i += 1;
                continue;
            }
            let Some(close_rel) = query[i + 1..].find('"') else {
                // Unbalanced quote: treat the dangling `"` as
                // whitespace (lenient, like lucene's parser) — close
                // the unquoted segment here and keep scanning after it.
                self.parse_unquoted_segment(&query[seg_start..i], &mut parsed);
                i += 1;
                seg_start = i;
                continue;
            };
            let close = i + 1 + close_rel;
            // A `+` / `-` glued to the opening quote — and itself at a
            // token boundary — sets the phrase's polarity; the sigil
            // byte is excluded from the unquoted segment.
            let sigil = match i > seg_start {
                true => {
                    let boundary = i - 1 == seg_start || bytes[i - 2].is_ascii_whitespace();
                    match (boundary, bytes[i - 1]) {
                        (true, b'+') => Some(b'+'),
                        (true, b'-') => Some(b'-'),
                        _ => None,
                    }
                }
                false => None,
            };
            let unquoted_end = match sigil {
                Some(_) => i - 1,
                None => i,
            };
            self.parse_unquoted_segment(&query[seg_start..unquoted_end], &mut parsed);
            let mut terms: Vec<Cow<'q, str>> = Vec::new();
            self.tokenize_each_query(&query[i + 1..close], &mut |t| terms.push(t));
            match (terms.len(), sigil) {
                // Empty quotes contribute nothing.
                (0, _) => {}
                // A single-token phrase is just that term — degrade to
                // the term list of the same polarity.
                (1, Some(b'-')) => parsed.negatives.push(terms.pop().expect("one term")),
                (1, Some(b'+')) => parsed.musts.push(terms.pop().expect("one term")),
                (1, _) => parsed.positives.push(terms.pop().expect("one term")),
                (_, Some(b'-')) => parsed.negative_phrases.push(terms),
                (_, Some(b'+')) => parsed.must_phrases.push(terms),
                (_, _) => parsed.positive_phrases.push(terms),
            }
            i = close + 1;
            seg_start = i;
        }
        self.parse_unquoted_segment(&query[seg_start..], &mut parsed);
        parsed
    }

    /// Parse one stretch of query text containing no quotes — the
    /// pre-phrase grammar: whitespace runs with optional `+`/`-`
    /// clause sigils.
    fn parse_unquoted_segment<'q>(&self, segment: &'q str, parsed: &mut ParsedQuery<'q>) {
        for run in segment.split_whitespace() {
            match (run.strip_prefix('-'), run.strip_prefix('+')) {
                (Some(rest), _) if !rest.is_empty() => {
                    self.tokenize_each_query(rest, &mut |t| parsed.negatives.push(t));
                }
                (_, Some(rest)) if !rest.is_empty() => {
                    self.tokenize_each_query(rest, &mut |t| parsed.musts.push(t));
                }
                _ => self.tokenize_each_query(run, &mut |t| parsed.positives.push(t)),
            }
        }
    }
}

/// Tokenize several `texts` into one sorted, de-duplicated term list.
/// For building a single term set from many values (e.g. an `IN` list)
/// where a word shared across values must be probed only once.
pub(crate) fn unique_tokens<'a>(
    tok: &dyn Tokenizer,
    texts: impl IntoIterator<Item = &'a str>,
) -> Vec<String> {
    texts
        .into_iter()
        .flat_map(|t| tok.tokenize(t))
        .collect::<BTreeSet<_>>()
        .into_iter()
        .collect()
}

/// ASCII whitespace + punctuation split, ASCII lowercase, no stemming,
/// no stopwords. The simplest tokenizer that's still useful.
#[derive(Debug, Clone, Copy, Default)]
pub struct AsciiLowerTokenizer;

impl AsciiLowerTokenizer {
    pub fn new() -> Self {
        Self
    }

    /// Zero-alloc emission with a borrowed fast path, generic over
    /// the callback type so LLVM can inline the callback body into
    /// the per-byte scan loop. Same shape as the trait
    /// [`Tokenizer::tokenize_each`] but takes `mut f: F` instead of
    /// `f: &mut dyn FnMut(&str)`, which:
    ///
    ///   * eliminates the per-token indirect dispatch on the
    ///     callback (~150M call sites on the 1M-doc bench);
    ///   * lets LLVM CSE common subexpressions between the
    ///     callback body (intern hash, dense_doc_tf load) and
    ///     the scan loop, and hoist invariants like the
    ///     interner / dense-array base pointers across iterations.
    ///
    /// The two byte-scan passes (skip non-token bytes; extend a
    /// token run) are SIMD-accelerated via [`simd_skip_non_token`]
    /// and [`simd_scan_token_run`] respectively — both process 16
    /// bytes per `u8x16` chunk on AVX2 hosts, replacing the
    /// previous one-byte-per-iteration scalar `while` loops. The
    /// bench corpus has ~2 KB / doc of token bytes, so the
    /// 16×-wide scan trades ~9.4M scalar branches per doc for
    /// ~590K `u8x16` chunk ops + a scalar tail.
    ///
    /// Scans the input once. For each token-byte run:
    ///   * If the run is **already lowercase ASCII** (the common case
    ///     for log lines, telemetry tokens, "term00042"-shaped Zipfian
    ///     bench corpora, and lower-cased ingestion pipelines) the
    ///     callback gets a borrowed `&str` slicing directly into the
    ///     input — zero copy, zero scratch-buf write.
    ///   * If the run contains uppercase ASCII bytes, the run is
    ///     copied into a reusable scratch `buf` while lower-casing in
    ///     place. The callback then gets `&buf`.
    ///   * If the run contains any non-ASCII byte (≥ 0x80), the whole
    ///     run is dropped per the v1 ASCII-only rule.
    ///
    /// The borrowed/copied `&str` is only valid for that one callback
    /// call. The next callback invocation may overwrite `buf` or hand
    /// out a different slice; copy via bumpalo/Box if you need to
    /// keep it.
    #[inline]
    pub fn tokenize_each_inline<F: FnMut(&str)>(&self, text: &str, mut f: F) {
        // One scan implementation — delegate and discard the position
        // ordinal so the positionless / query paths and the positional
        // index build can never disagree on which tokens are emitted.
        self.tokenize_each_inline_positioned(text, |tok, _position| f(tok));
    }

    /// Like [`Self::tokenize_each_inline`], but also hands each emitted
    /// token its **position ordinal**: the count of token runs scanned
    /// before it, *including runs this tokenizer drops*.
    ///
    /// A run that contains a non-ASCII byte is dropped (no token), but
    /// it still consumes one ordinal — so a phrase never treats the
    /// tokens on either side of a dropped word as adjacent. Without
    /// this, `"new café york"` would tokenize to `new`, `york` at
    /// positions 0, 1 and the phrase `"new york"` would wrongly match
    /// it; with the gap they sit at 0 and 2. Used by the positional
    /// index build. (A run consisting *only* of non-ASCII bytes carries
    /// no ASCII token byte to anchor the scan and is treated as a
    /// separator, not a gap — the ASCII tokenizer cannot represent such
    /// text at all.)
    ///
    /// The borrowed/copied `&str` is valid only for that one callback
    /// call, exactly as in [`Self::tokenize_each_inline`].
    #[inline]
    pub fn tokenize_each_inline_positioned<F: FnMut(&str, u64)>(&self, text: &str, mut f: F) {
        let bytes = text.as_bytes();
        let mut buf: Vec<u8> = Vec::new();
        let mut pos = 0;
        let mut position: u64 = 0;
        while pos < bytes.len() {
            pos = simd_skip_non_token(bytes, pos);
            if pos >= bytes.len() {
                return;
            }
            let start = pos;
            let (end, had_upper, had_non_ascii) = simd_scan_token_run(bytes, pos);
            pos = end;
            if start == pos {
                continue;
            }
            // Every scanned run occupies one ordinal, dropped or not.
            let this_position = position;
            position += 1;
            if had_non_ascii {
                // Dropped per the v1 ASCII-only rule — the ordinal it
                // just consumed is the phrase gap it leaves behind.
                continue;
            }
            if !had_upper {
                // Fast path: borrow directly from `text`.
                //
                // SAFETY: `is_token_byte` only accepts ASCII
                // alphanumerics, so every byte in `bytes[start..end]`
                // is a single-byte ASCII codepoint. The slice is
                // therefore valid UTF-8 and the original `text`
                // outlives the callback call.
                let s = unsafe { from_utf8_unchecked(&bytes[start..end]) };
                f(s, this_position);
            } else {
                // Slow path: copy + lowercase into the reusable buf.
                buf.clear();
                buf.reserve(end - start);
                for &b in &bytes[start..end] {
                    buf.push(b.to_ascii_lowercase());
                }
                // SAFETY: same reasoning — every byte pushed is an
                // ASCII alphanumeric (or its lowercased form, which
                // is also ASCII).
                let s = unsafe { from_utf8_unchecked(&buf) };
                f(s, this_position);
            }
        }
    }
}

/// SIMD scan: advance `pos` past non-token bytes, returning the
/// index of the first ASCII alphanumeric byte (or `bytes.len()`).
///
/// Replaces a per-byte `while !is_ascii_alphanumeric(bytes[pos])`
/// loop with a 16-byte `u8x16` chunked scan. Within each chunk we
/// build an "is token byte" mask (`'0'..='9' | 'A'..='Z' |
/// 'a'..='z'`) via three range comparisons + two ORs, then jump to
/// the first set bit via `trailing_zeros`. Falls back to a scalar
/// tail for `bytes.len() % 16` trailing bytes.
#[inline(always)]
fn simd_skip_non_token(bytes: &[u8], mut pos: usize) -> usize {
    const LANES: usize = 16;
    while pos + LANES <= bytes.len() {
        // SAFETY: `pos + LANES <= bytes.len()` was checked above,
        // so reading 16 bytes from `bytes.as_ptr().add(pos)` stays
        // in-bounds. The cast to `*const [u8; LANES]` and deref
        // produces a copy (the array is loaded by value into the
        // SIMD register on the next line).
        let arr: [u8; LANES] = unsafe { *(bytes.as_ptr().add(pos) as *const [u8; LANES]) };
        let chunk = u8x16::from(arr);
        let is_digit = chunk.simd_ge(u8x16::splat(b'0')) & chunk.simd_le(u8x16::splat(b'9'));
        let is_upper = chunk.simd_ge(u8x16::splat(b'A')) & chunk.simd_le(u8x16::splat(b'Z'));
        let is_lower = chunk.simd_ge(u8x16::splat(b'a')) & chunk.simd_le(u8x16::splat(b'z'));
        let is_token = is_digit | is_upper | is_lower;
        let mask = is_token.to_bitmask() & LANE_BITMASK;
        if mask == 0 {
            pos += LANES;
        } else {
            return pos + mask.trailing_zeros() as usize;
        }
    }
    // Scalar tail.
    while pos < bytes.len() && !bytes[pos].is_ascii_alphanumeric() {
        pos += 1;
    }
    pos
}

/// SIMD scan: extend a token-byte run starting at `pos`, returning
/// `(end, had_upper, had_non_ascii)`. The run extends as long as
/// each byte is either an ASCII alphanumeric **or** a non-ASCII
/// (high-bit) byte — the latter just sets the `had_non_ascii`
/// drop-marker per the v1 ASCII-only rule. The run stops on the
/// first ASCII separator (any non-alphanumeric byte `< 0x80`).
///
/// Equivalent to the scalar version above but with 16-byte
/// chunked compares. Within each chunk:
///   * build the "extend" mask (`is_token | is_high`);
///   * if all 16 lanes extend, OR-in the `had_upper` /
///     `had_non_ascii` flags from the full chunk and advance 16;
///   * otherwise find the first separator lane via
///     `trailing_zeros(!extend & 0xFFFF)`, mask the flag bitmasks
///     to the consumed prefix only (so flags from bytes past the
///     separator don't leak into this token), and return.
#[inline(always)]
fn simd_scan_token_run(bytes: &[u8], mut pos: usize) -> (usize, bool, bool) {
    const LANES: usize = 16;
    let mut had_upper = false;
    let mut had_non_ascii = false;
    while pos + LANES <= bytes.len() {
        // SAFETY: bounds-checked at the loop guard above.
        let arr: [u8; LANES] = unsafe { *(bytes.as_ptr().add(pos) as *const [u8; LANES]) };
        let chunk = u8x16::from(arr);
        let is_digit = chunk.simd_ge(u8x16::splat(b'0')) & chunk.simd_le(u8x16::splat(b'9'));
        let is_upper = chunk.simd_ge(u8x16::splat(b'A')) & chunk.simd_le(u8x16::splat(b'Z'));
        let is_lower = chunk.simd_ge(u8x16::splat(b'a')) & chunk.simd_le(u8x16::splat(b'z'));
        // High-bit detect: mask high bit, compare equal to 0x80.
        // `simd_eq` is bit-equality on signed/unsigned-agnostic
        // `cmp_eq_mask_i8_m128i`, so this works for high bytes
        // even though `simd_gt`/`simd_ge` against `0x80` would
        // need an XOR-flip trick to handle signed-i8 wrap.
        let is_high =
            (chunk & u8x16::splat(NON_ASCII_BYTE_MIN)).simd_eq(u8x16::splat(NON_ASCII_BYTE_MIN));
        let is_token = is_digit | is_upper | is_lower;
        let is_extend = is_token | is_high;
        let extend_mask = is_extend.to_bitmask() & LANE_BITMASK;
        let upper_mask = is_upper.to_bitmask() & LANE_BITMASK;
        let high_mask = is_high.to_bitmask() & LANE_BITMASK;
        let non_extend = !extend_mask & LANE_BITMASK;
        if non_extend == 0 {
            had_upper |= upper_mask != 0;
            had_non_ascii |= high_mask != 0;
            pos += LANES;
        } else {
            let sep_idx = non_extend.trailing_zeros() as usize;
            let prefix_mask: u32 = (1u32 << sep_idx).wrapping_sub(1);
            had_upper |= (upper_mask & prefix_mask) != 0;
            had_non_ascii |= (high_mask & prefix_mask) != 0;
            pos += sep_idx;
            return (pos, had_upper, had_non_ascii);
        }
    }
    // Scalar tail (same logic as the scalar version).
    while pos < bytes.len() {
        let b = bytes[pos];
        if is_token_byte(b) {
            had_upper |= b.is_ascii_uppercase();
            pos += 1;
        } else if b >= NON_ASCII_BYTE_MIN {
            had_non_ascii = true;
            pos += 1;
        } else {
            break;
        }
    }
    (pos, had_upper, had_non_ascii)
}

/// A parsed BM25 query, split into its clause lists by leading sigil:
/// `+term` → `musts`, bare `term` → `positives`, `-term` →
/// `negatives`. Tokens may borrow the query string, so this can't
/// outlive the query.
#[derive(Debug, Default)]
pub struct ParsedQuery<'q> {
    /// `+`-sigiled tokens: the doc must contain every one.
    pub musts: Vec<Cow<'q, str>>,
    /// Bare (sigil-less) tokens. Their polarity comes from the
    /// default operator: [`BoolMode::And`] treats them as musts,
    /// [`BoolMode::Or`] as shoulds (scoring-only once any must
    /// exists; a plain union when none does).
    pub positives: Vec<Cow<'q, str>>,
    /// `-`-sigiled tokens: any doc containing one is excluded.
    pub negatives: Vec<Cow<'q, str>>,
    /// `+"…"`-quoted runs of two or more tokens: the doc must contain
    /// the exact token sequence. (Single-token phrases degrade into
    /// `musts`.)
    pub must_phrases: Vec<Vec<Cow<'q, str>>>,
    /// Bare-quoted multi-token runs; polarity resolved from the
    /// default operator like bare terms.
    pub positive_phrases: Vec<Vec<Cow<'q, str>>>,
    /// `-"…"`-quoted multi-token runs: any doc containing the exact
    /// sequence is excluded.
    pub negative_phrases: Vec<Vec<Cow<'q, str>>>,
}

/// A query's clause lists with the default operator already applied —
/// what [`ParsedQuery::into_clauses`] produces and the search kernels
/// consume. `shoulds` is non-empty only under [`BoolMode::Or`].
#[derive(Debug, Default)]
pub struct QueryClauses<'q> {
    /// Every doc in the result must contain all of these.
    pub musts: Vec<Cow<'q, str>>,
    /// Scoring-only when `musts` is non-empty; otherwise the match is
    /// their union.
    pub shoulds: Vec<Cow<'q, str>>,
    /// Docs containing any of these are excluded.
    pub negatives: Vec<Cow<'q, str>>,
    /// Multi-token phrases every doc in the result must contain.
    pub must_phrases: Vec<Vec<Cow<'q, str>>>,
    /// Scoring-only phrases when `musts`/`must_phrases` is non-empty;
    /// otherwise part of the union match.
    pub should_phrases: Vec<Vec<Cow<'q, str>>>,
    /// Docs containing any of these exact sequences are excluded.
    pub negative_phrases: Vec<Vec<Cow<'q, str>>>,
}

impl<'q> ParsedQuery<'q> {
    /// Resolve the bare tokens' polarity from the default operator
    /// `mode`: `And` folds them into `musts`, `Or` makes them
    /// `shoulds`. Sigiled tokens keep their explicit polarity.
    pub fn into_clauses(self, mode: BoolMode) -> QueryClauses<'q> {
        let ParsedQuery {
            mut musts,
            positives,
            negatives,
            mut must_phrases,
            positive_phrases,
            negative_phrases,
        } = self;
        let shoulds = match mode {
            BoolMode::And => {
                musts.extend(positives);
                Vec::new()
            }
            BoolMode::Or => positives,
        };
        let should_phrases = match mode {
            BoolMode::And => {
                must_phrases.extend(positive_phrases);
                Vec::new()
            }
            BoolMode::Or => positive_phrases,
        };
        QueryClauses {
            musts,
            shoulds,
            negatives,
            must_phrases,
            should_phrases,
            negative_phrases,
        }
    }
}

impl Tokenizer for AsciiLowerTokenizer {
    fn name(&self) -> &'static str {
        ASCII_LOWER_TOKENIZER
    }

    fn tokenize<'a>(&'a self, text: &'a str) -> Box<dyn Iterator<Item = String> + 'a> {
        Box::new(AsciiLowerIter::new(text.as_bytes()))
    }

    /// Trait-object dispatch path: delegates to the inherent
    /// [`tokenize_each_inline`](Self::tokenize_each_inline) so the
    /// body lives in one place. Callers that hold a concrete
    /// [`AsciiLowerTokenizer`] (or successfully downcast a `&dyn
    /// Tokenizer` via [`Tokenizer::as_any`]) should call the
    /// inherent method directly to skip the per-token
    /// `&mut dyn FnMut(&str)` indirection.
    fn tokenize_each(&self, text: &str, f: &mut dyn FnMut(&str)) {
        self.tokenize_each_inline(text, |s| f(s));
    }

    fn as_any(&self) -> &dyn Any {
        self
    }

    /// Zero-copy override: an already-lowercase token borrows from
    /// `text`; only a token that needs lowercasing is copied.
    fn tokenize_each_query<'q>(&self, text: &'q str, f: &mut dyn FnMut(Cow<'q, str>)) {
        let bytes = text.as_bytes();
        let mut pos = 0;
        while pos < bytes.len() {
            pos = simd_skip_non_token(bytes, pos);
            if pos >= bytes.len() {
                return;
            }
            let start = pos;
            let (end, had_upper, had_non_ascii) = simd_scan_token_run(bytes, pos);
            pos = end;
            if had_non_ascii || start == pos {
                continue;
            }
            let s = from_utf8(&bytes[start..end]).expect("ASCII-only by construction");
            if had_upper {
                f(Cow::Owned(s.to_ascii_lowercase()));
            } else {
                f(Cow::Borrowed(s));
            }
        }
    }
}

/// Internal iterator that walks the input byte slice once, emitting
/// lowercased tokens. Skips tokens containing non-ASCII bytes per the
/// v1 ASCII-only rule.
struct AsciiLowerIter<'a> {
    src: &'a [u8],
    pos: usize,
    buf: Vec<u8>,
}

impl<'a> AsciiLowerIter<'a> {
    fn new(src: &'a [u8]) -> Self {
        Self {
            src,
            pos: 0,
            buf: Vec::with_capacity(TOKEN_SCRATCH_INITIAL_CAP),
        }
    }
}

impl Iterator for AsciiLowerIter<'_> {
    type Item = String;

    fn next(&mut self) -> Option<String> {
        loop {
            // Skip non-token bytes.
            while self.pos < self.src.len() && !is_token_byte(self.src[self.pos]) {
                self.pos += 1;
            }
            if self.pos >= self.src.len() {
                return None;
            }

            // Accumulate one token.
            self.buf.clear();
            let mut had_non_ascii = false;
            while self.pos < self.src.len() {
                let b = self.src[self.pos];
                if is_token_byte(b) {
                    self.buf.push(b.to_ascii_lowercase());
                    self.pos += 1;
                } else if b >= NON_ASCII_BYTE_MIN {
                    // Non-ASCII byte inside a contiguous "word-ish" run —
                    // mark this run as non-ASCII and consume until a true
                    // separator. Drop the whole token.
                    had_non_ascii = true;
                    self.pos += 1;
                } else {
                    break;
                }
            }

            if had_non_ascii || self.buf.is_empty() {
                continue;
            }

            // SAFETY: we only push ASCII letters and digits via
            // is_token_byte + to_ascii_lowercase, so the buffer is
            // guaranteed valid UTF-8.
            let s = from_utf8(&self.buf)
                .expect("ASCII-only by construction")
                .to_owned();
            return Some(s);
        }
    }
}

/// `[A-Za-z0-9]` — the v1 token alphabet.
#[inline]
fn is_token_byte(b: u8) -> bool {
    b.is_ascii_alphanumeric()
}

/// Name of the default ASCII tokenizer in a column's FTS config.
pub const ASCII_LOWER_TOKENIZER: &str = "ascii_lower";

/// Name of the Unicode-aware standard tokenizer in a column's FTS config.
pub const STANDARD_TOKENIZER: &str = "standard";

/// Resolve a tokenizer name to an instance, or `None` for an
/// unrecognized name. The single routing point shared by the build
/// path (mapping a chosen analyzer to the tokenizer used at index
/// time) and the read path (reconstructing a column's tokenizer from
/// the name recorded in its stored config). Callers translate `None`
/// into their own error — a malformed-superfile read error, or an
/// invalid-argument error at table-create time.
pub fn tokenizer_for_name(name: &str) -> Option<Arc<dyn Tokenizer>> {
    match name {
        ASCII_LOWER_TOKENIZER => Some(Arc::new(AsciiLowerTokenizer)),
        STANDARD_TOKENIZER => Some(Arc::new(StandardTokenizer)),
        _ => None,
    }
}

/// Unicode-aware tokenizer: UAX #29 word segmentation followed by full
/// Unicode lowercasing, preserving non-ASCII text. See the module-level
/// docs for the exact semantics and how it differs from
/// [`AsciiLowerTokenizer`].
#[derive(Debug, Clone, Copy, Default)]
pub struct StandardTokenizer;

impl StandardTokenizer {
    pub fn new() -> Self {
        Self
    }
}

impl Tokenizer for StandardTokenizer {
    fn name(&self) -> &'static str {
        STANDARD_TOKENIZER
    }

    fn tokenize<'a>(&'a self, text: &'a str) -> Box<dyn Iterator<Item = String> + 'a> {
        // `unicode_words` yields the UAX #29 word segments that contain
        // alphanumerics — whitespace/punctuation-only segments are
        // dropped. Lowercasing is full Unicode case folding, correct for
        // non-ASCII letters, which are kept rather than dropped.
        Box::new(text.unicode_words().map(str::to_lowercase))
    }

    fn tokenize_each(&self, text: &str, f: &mut dyn FnMut(&str)) {
        let mut buf = String::new();
        for word in text.unicode_words() {
            // Borrow directly when every cased character is already
            // lowercase (the common case for lowercased corpora); only
            // allocate to case-fold a word carrying an upper/title-case
            // letter. Non-alphabetic characters (digits, apostrophes)
            // are unaffected by lowercasing, so they never force a copy.
            if word.chars().all(|c| !c.is_alphabetic() || c.is_lowercase()) {
                f(word);
            } else {
                buf.clear();
                // Context-aware full-string lowercasing, matching
                // `tokenize`. `str::to_lowercase` applies Unicode
                // special-casing such as Final_Sigma (a word-final `Σ`
                // lowercases to `ς`, but to `σ` elsewhere); a char-by-char
                // fold has no word context and would emit `σ` in both
                // spots. The two paths must agree — text is indexed
                // through `tokenize_each` and queried through `tokenize`,
                // so any divergence indexes a term under one form and
                // searches it under another.
                buf.push_str(&word.to_lowercase());
                f(&buf);
            }
        }
    }

    fn as_any(&self) -> &dyn Any {
        self
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn tokens(text: &str) -> Vec<String> {
        AsciiLowerTokenizer.tokenize(text).collect()
    }

    /// Collect `(token, position)` pairs from the positional scan.
    fn positioned(text: &str) -> Vec<(String, u64)> {
        let mut out = Vec::new();
        AsciiLowerTokenizer
            .tokenize_each_inline_positioned(text, |tok, pos| out.push((tok.to_owned(), pos)));
        out
    }

    // ---- StandardTokenizer (Unicode-aware) ----

    /// Tokens via the trait `tokenize` path.
    fn std_tokens(text: &str) -> Vec<String> {
        StandardTokenizer.tokenize(text).collect()
    }

    /// Tokens via the `tokenize_each` borrowing path — must agree with
    /// `tokenize` on which tokens are emitted.
    fn std_tokens_each(text: &str) -> Vec<String> {
        let mut out = Vec::new();
        StandardTokenizer.tokenize_each(text, &mut |t| out.push(t.to_owned()));
        out
    }

    #[test]
    fn standard_lowercases_and_splits_ascii() {
        assert_eq!(
            std_tokens("Rust Async Runtime"),
            vec!["rust", "async", "runtime"]
        );
        assert_eq!(
            std_tokens_each("Rust Async Runtime"),
            vec!["rust", "async", "runtime"]
        );
    }

    #[test]
    fn standard_keeps_non_ascii_lowercased() {
        // The key divergence from AsciiLowerTokenizer, which drops these.
        assert_eq!(std_tokens("Café RÉSUMÉ"), vec!["café", "résumé"]);
        assert_eq!(std_tokens_each("Café RÉSUMÉ"), vec!["café", "résumé"]);
        // AsciiLowerTokenizer drops the same tokens entirely.
        assert_eq!(
            AsciiLowerTokenizer
                .tokenize("Café RÉSUMÉ")
                .collect::<Vec<_>>(),
            Vec::<String>::new()
        );
    }

    #[test]
    fn standard_splits_cjk_per_ideograph() {
        // UAX #29 treats each CJK ideograph as its own word.
        assert_eq!(std_tokens("日本語"), vec!["日", "本", "語"]);
    }

    #[test]
    fn standard_keeps_intra_word_numeric_and_apostrophe() {
        // UAX #29 keeps a decimal number and a mid-word apostrophe together.
        assert_eq!(std_tokens("pi is 3.14"), vec!["pi", "is", "3.14"]);
        assert_eq!(std_tokens("don't stop"), vec!["don't", "stop"]);
    }

    #[test]
    fn standard_splits_on_hyphen_and_drops_punctuation() {
        assert_eq!(std_tokens("wi-fi, hello!"), vec!["wi", "fi", "hello"]);
        assert_eq!(std_tokens("...   ???"), Vec::<String>::new());
    }

    #[test]
    fn standard_borrow_and_copy_paths_agree() {
        // Mixed already-lower, needs-fold, digit, and non-ASCII tokens:
        // the borrow fast path and the copy path must emit the same set.
        let text = "alpha Beta 42 gamma2 Δelta";
        assert_eq!(std_tokens(text), std_tokens_each(text));
    }

    #[test]
    fn standard_copy_path_lowercases_final_sigma_like_tokenize() {
        // Regression: the copy path must lowercase with context-aware
        // `str::to_lowercase`, not char-by-char. A word-final capital `Σ`
        // folds to `ς` (final sigma) but to `σ` elsewhere; a char-by-char
        // fold has no word context and would emit `σ` in both places. Text
        // is indexed through `tokenize_each` and queried through
        // `tokenize`, so any divergence stores a Greek term under one form
        // and searches it under another — the document becomes unfindable.
        let text = "ΟΔΟΣ"; // one Greek word; the final Σ must fold to ς
        // Both tokenizer paths agree, and both equal Rust's context-aware
        // folding (which the char-by-char version would not).
        assert_eq!(std_tokens(text), std_tokens_each(text));
        assert_eq!(std_tokens_each(text), vec![text.to_lowercase()]);
        assert!(
            std_tokens_each(text)[0].ends_with('ς'),
            "word-final Σ must fold to final sigma ς, not σ"
        );
    }

    #[test]
    fn standard_empty_and_whitespace_yield_nothing() {
        assert_eq!(std_tokens(""), Vec::<String>::new());
        assert_eq!(std_tokens("   \t\n"), Vec::<String>::new());
    }

    #[test]
    fn standard_query_parse_keeps_non_ascii_and_sigils() {
        // Query-side tokenization (via the default `tokenize_each_query`
        // → `parse`) must keep non-ASCII and honor +/- clause sigils.
        let p = StandardTokenizer.parse("Café -Résumé +Ötzi");
        assert_eq!(p.positives, vec!["café"]);
        assert_eq!(p.negatives, vec!["résumé"]);
        assert_eq!(p.musts, vec!["ötzi"]);
    }

    #[test]
    fn tokenizer_for_name_resolves_known_and_rejects_unknown() {
        assert!(tokenizer_for_name(ASCII_LOWER_TOKENIZER).is_some());
        assert!(tokenizer_for_name(STANDARD_TOKENIZER).is_some());
        assert!(tokenizer_for_name("nonesuch").is_none());
        // The resolved standard tokenizer keeps non-ASCII through the
        // trait object, confirming the right impl is wired.
        let tok = tokenizer_for_name(STANDARD_TOKENIZER).expect("standard");
        assert_eq!(tok.tokenize("Café").collect::<Vec<_>>(), vec!["café"]);
    }

    #[test]
    fn positioned_leaves_a_gap_for_dropped_runs() {
        // No drops: positions are dense emission ordinals, and the
        // positioned scan emits exactly the same tokens as the plain
        // one (delegation guarantees this).
        assert_eq!(
            positioned("the quick brown fox"),
            vec![
                ("the".into(), 0),
                ("quick".into(), 1),
                ("brown".into(), 2),
                ("fox".into(), 3),
            ],
        );

        // A dropped (non-ASCII) run consumes an ordinal but emits no
        // token, so the neighbours are NOT adjacent: `york` is at 2,
        // not 1 — this is what stops `"new york"` matching this text.
        assert_eq!(
            positioned("new café york"),
            vec![("new".into(), 0), ("york".into(), 2)],
        );
        // Gap whether the dropped word leads, trails, or sits between.
        assert_eq!(
            positioned("café new york"),
            vec![("new".into(), 1), ("york".into(), 2)],
        );
        assert_eq!(
            positioned("new york café"),
            vec![("new".into(), 0), ("york".into(), 1)],
        );

        // The plain scan still emits the same tokens (positions dropped).
        let mut plain = Vec::new();
        AsciiLowerTokenizer.tokenize_each_inline("new café york", |t| plain.push(t.to_owned()));
        assert_eq!(plain, vec!["new".to_string(), "york".to_string()]);
    }

    #[test]
    fn empty_input_yields_nothing() {
        assert_eq!(tokens(""), Vec::<String>::new());
    }

    #[test]
    fn whitespace_only_yields_nothing() {
        assert_eq!(tokens("   \t\n\r"), Vec::<String>::new());
    }

    #[test]
    fn single_token_lowercased() {
        assert_eq!(tokens("Hello"), vec!["hello"]);
    }

    #[test]
    fn unique_tokens_dedups_and_sorts_across_values() {
        let tok = AsciiLowerTokenizer;
        // values share 'juice'; result is one sorted set, no repeat
        let got = unique_tokens(&tok, ["Orange Juice", "Apple Juice"]);
        assert_eq!(got, vec!["apple", "juice", "orange"]);
    }

    #[test]
    fn multiple_tokens_split_on_whitespace() {
        assert_eq!(
            tokens("Rust async runtime"),
            vec!["rust", "async", "runtime"]
        );
    }

    #[test]
    fn punctuation_splits_tokens() {
        assert_eq!(
            tokens("hello,world!foo;bar.baz?"),
            vec!["hello", "world", "foo", "bar", "baz"]
        );
    }

    #[test]
    fn case_folding_applies_to_uppercase_only() {
        assert_eq!(tokens("ABC abc XyZ"), vec!["abc", "abc", "xyz"]);
    }

    #[test]
    fn alphanumerics_kept_together() {
        assert_eq!(tokens("foo123 bar456"), vec!["foo123", "bar456"]);
    }

    #[test]
    fn pure_numeric_tokens_kept() {
        assert_eq!(tokens("404 200 500"), vec!["404", "200", "500"]);
    }

    #[test]
    fn underscore_is_a_separator_in_v1() {
        // `_` is not in `[A-Za-z0-9]` — it splits tokens. v2 may revisit.
        assert_eq!(tokens("foo_bar"), vec!["foo", "bar"]);
    }

    #[test]
    fn dash_is_a_separator() {
        assert_eq!(tokens("rust-async"), vec!["rust", "async"]);
    }

    #[test]
    fn non_ascii_token_is_dropped() {
        // ASCII-only tokenizer: "café" has a non-ASCII byte, so the
        // entire token is dropped.
        assert_eq!(tokens("café"), Vec::<String>::new());
    }

    #[test]
    fn non_ascii_token_drops_only_that_token() {
        // Surrounding ASCII tokens still come through.
        assert_eq!(tokens("hello café world"), vec!["hello", "world"]);
    }

    #[test]
    fn cjk_input_yields_nothing() {
        assert_eq!(tokens("日本語"), Vec::<String>::new());
    }

    #[test]
    fn emoji_input_yields_nothing() {
        assert_eq!(tokens("hello 🚀 world"), vec!["hello", "world"]);
    }

    #[test]
    fn multiple_consecutive_separators_are_collapsed() {
        assert_eq!(tokens("foo,,,bar"), vec!["foo", "bar"]);
        assert_eq!(tokens("foo   bar"), vec!["foo", "bar"]);
    }

    #[test]
    fn leading_and_trailing_separators_are_skipped() {
        assert_eq!(tokens("  foo bar  "), vec!["foo", "bar"]);
        assert_eq!(tokens("...foo..."), vec!["foo"]);
    }

    #[test]
    fn tokenizer_is_send_and_sync() {
        // Compile-time assertion via the Tokenizer trait bound.
        fn is_send_sync<T: Send + Sync>() {}
        is_send_sync::<AsciiLowerTokenizer>();
    }

    #[test]
    fn tokenizer_used_via_dyn_trait() {
        // The trait object form is what the FtsBuilder will hold.
        let tok: Box<dyn Tokenizer> = Box::new(AsciiLowerTokenizer);
        let v: Vec<String> = tok.tokenize("Hello WORLD").collect();
        assert_eq!(v, vec!["hello", "world"]);
    }

    #[test]
    fn stress_long_input_does_not_panic() {
        // Rough scale-test: 1 MB of pseudo-text.
        let chunk = "lorem ipsum dolor sit amet, consectetur adipiscing elit. ";
        let big = chunk.repeat(20_000);
        let count = AsciiLowerTokenizer.tokenize(&big).count();
        // 8 tokens per chunk × 20_000 = 160_000.
        assert_eq!(count, 8 * 20_000);
    }

    // ---- parse (the `-` negation sigil) ----

    fn parse(query: &str) -> ParsedQuery<'_> {
        AsciiLowerTokenizer.parse(query)
    }

    #[test]
    fn parse_default_trait_impl_matches_override() {
        // A tokenizer that overrides nothing gets the same split via
        // the default `parse` impl (owned tokens).
        #[derive(Debug)]
        struct PlainTok;
        impl Tokenizer for PlainTok {
            fn name(&self) -> &'static str {
                "plain_test"
            }
            fn tokenize<'a>(&'a self, text: &'a str) -> Box<dyn Iterator<Item = String> + 'a> {
                AsciiLowerTokenizer.tokenize(text)
            }
            fn as_any(&self) -> &dyn Any {
                self
            }
        }
        let p = PlainTok.parse("Rust -PYTHON");
        assert_eq!(p.positives, vec!["rust"]);
        assert_eq!(p.negatives, vec!["python"]);
        assert!(matches!(p.positives[0], Cow::Owned(_)));
    }

    #[test]
    fn parse_positives_only() {
        let p = parse("rust async");
        assert_eq!(p.positives, vec!["rust", "async"]);
        assert!(p.negatives.is_empty());
    }

    #[test]
    fn parse_single_negative() {
        let p = parse("rust -python");
        assert_eq!(p.positives, vec!["rust"]);
        assert_eq!(p.negatives, vec!["python"]);
    }

    #[test]
    fn parse_multiple_negatives() {
        let p = parse("rust async -python -php");
        assert_eq!(p.positives, vec!["rust", "async"]);
        assert_eq!(p.negatives, vec!["python", "php"]);
    }

    #[test]
    fn parse_negation_only() {
        // No positive clause — the parser reports it faithfully; the
        // caller turns this into an error.
        let p = parse("-python");
        assert!(p.positives.is_empty());
        assert_eq!(p.negatives, vec!["python"]);
    }

    #[test]
    fn parse_interior_hyphen_is_not_negation() {
        // `a-b` is one run with an interior `-`; the scan splits it
        // into two positive tokens. Nothing is negated.
        let p = parse("a-b");
        assert_eq!(p.positives, vec!["a", "b"]);
        assert!(p.negatives.is_empty());
    }

    #[test]
    fn parse_bare_dash_contributes_nothing() {
        let p = parse("rust - python");
        assert_eq!(p.positives, vec!["rust", "python"]);
        assert!(p.negatives.is_empty());
    }

    #[test]
    fn parse_double_dash_strips_one_then_tokenizes() {
        // `--py`: strip the one leading `-`, leaving `-py`; the scan
        // drops the remaining `-` and yields `py`.
        let p = parse("--py");
        assert!(p.positives.is_empty());
        assert_eq!(p.negatives, vec!["py"]);
    }

    #[test]
    fn parse_negated_term_is_normalized() {
        // The negated side is lower-cased like the index.
        let p = parse("rust -PYTHON");
        assert_eq!(p.negatives, vec!["python"]);
    }

    #[test]
    fn parse_empty_query() {
        let p = parse("");
        assert!(p.musts.is_empty());
        assert!(p.positives.is_empty());
        assert!(p.negatives.is_empty());
    }

    // ---- parse (the `+` must sigil) ----

    #[test]
    fn parse_must_sigil() {
        let p = parse("+climate policy");
        assert_eq!(p.musts, vec!["climate"]);
        assert_eq!(p.positives, vec!["policy"]);
        assert!(p.negatives.is_empty());
    }

    #[test]
    fn parse_all_must() {
        let p = parse("+griffith +observatory");
        assert_eq!(p.musts, vec!["griffith", "observatory"]);
        assert!(p.positives.is_empty());
    }

    #[test]
    fn parse_must_with_negation() {
        let p = parse("+python -snake -monty");
        assert_eq!(p.musts, vec!["python"]);
        assert!(p.positives.is_empty());
        assert_eq!(p.negatives, vec!["snake", "monty"]);
    }

    #[test]
    fn parse_interior_plus_is_not_must() {
        // `a+b` is one run with an interior `+`; the scan splits it
        // into two bare tokens. Nothing is a must clause.
        let p = parse("a+b");
        assert!(p.musts.is_empty());
        assert_eq!(p.positives, vec!["a", "b"]);
    }

    #[test]
    fn parse_bare_plus_contributes_nothing() {
        let p = parse("rust + python");
        assert!(p.musts.is_empty());
        assert_eq!(p.positives, vec!["rust", "python"]);
    }

    #[test]
    fn parse_must_term_is_normalized() {
        // The must side is lower-cased like the index.
        let p = parse("+RUST async");
        assert_eq!(p.musts, vec!["rust"]);
        assert_eq!(p.positives, vec!["async"]);
    }

    #[test]
    fn parse_minus_wins_over_plus_ordering() {
        // `-` is checked first, so `-+x` negates (strip `-`, the scan
        // drops the `+`); `+-x` is a must (strip `+`, scan drops `-`).
        let p = parse("-+x");
        assert_eq!(p.negatives, vec!["x"]);
        let p = parse("+-x");
        assert_eq!(p.musts, vec!["x"]);
    }

    // ---- parse (quoted phrase atoms) ----

    #[test]
    fn parse_pure_phrase() {
        let p = parse(r#""griffith observatory""#);
        assert_eq!(p.positive_phrases, vec![vec!["griffith", "observatory"]]);
        assert!(p.positives.is_empty());
        assert!(p.musts.is_empty());
    }

    #[test]
    fn parse_phrase_polarities() {
        let p = parse(r#"+"the who" -"memory unsafe" "new york""#);
        assert_eq!(p.must_phrases, vec![vec!["the", "who"]]);
        assert_eq!(p.negative_phrases, vec![vec!["memory", "unsafe"]]);
        assert_eq!(p.positive_phrases, vec![vec!["new", "york"]]);
    }

    #[test]
    fn parse_phrase_mixes_with_terms() {
        let p = parse(r#"+"the who" +uk rust -python"#);
        assert_eq!(p.must_phrases, vec![vec!["the", "who"]]);
        assert_eq!(p.musts, vec!["uk"]);
        assert_eq!(p.positives, vec!["rust"]);
        assert_eq!(p.negatives, vec!["python"]);
    }

    #[test]
    fn parse_single_token_phrase_degrades_to_term() {
        let p = parse(r#""york" +"london" -"paris""#);
        assert!(p.positive_phrases.is_empty());
        assert!(p.must_phrases.is_empty());
        assert!(p.negative_phrases.is_empty());
        assert_eq!(p.positives, vec!["york"]);
        assert_eq!(p.musts, vec!["london"]);
        assert_eq!(p.negatives, vec!["paris"]);
    }

    #[test]
    fn parse_empty_quotes_contribute_nothing() {
        let p = parse(r#"rust "" async"#);
        assert_eq!(p.positives, vec!["rust", "async"]);
        assert!(p.positive_phrases.is_empty());
    }

    #[test]
    fn parse_unbalanced_quote_is_whitespace() {
        // The dangling quote splits the text; everything parses as
        // bare terms (lenient, never an error).
        let p = parse(r#"rust "new york"#);
        assert_eq!(p.positives, vec!["rust", "new", "york"]);
        assert!(p.positive_phrases.is_empty());
    }

    #[test]
    fn parse_phrase_tokens_are_normalized() {
        // Phrase innards run through the same tokenizer: lowercased,
        // punctuation split.
        let p = parse(r#""New-York City""#);
        assert_eq!(p.positive_phrases, vec![vec!["new", "york", "city"]]);
    }

    #[test]
    fn parse_interior_sigil_before_quote_is_not_polarity() {
        // `abc+"x y"`: the `+` is interior to the run, not a phrase
        // sigil — the phrase is bare and `abc` parses from the
        // unquoted segment (its trailing `+` strips as punctuation).
        let p = parse(r#"abc+"x y""#);
        assert_eq!(p.positives, vec!["abc"]);
        assert_eq!(p.positive_phrases, vec![vec!["x", "y"]]);
        assert!(p.must_phrases.is_empty());
    }

    #[test]
    fn parse_adjacent_phrases() {
        let p = parse(r#""a b""c d""#);
        assert_eq!(p.positive_phrases, vec![vec!["a", "b"], vec!["c", "d"]]);
    }

    #[test]
    fn into_clauses_resolves_phrase_polarity_by_mode() {
        let c = parse(r#""new york" +"the who" -"bad seq" rust"#).into_clauses(BoolMode::Or);
        assert_eq!(c.should_phrases, vec![vec!["new", "york"]]);
        assert_eq!(c.must_phrases, vec![vec!["the", "who"]]);
        assert_eq!(c.negative_phrases, vec![vec!["bad", "seq"]]);
        assert_eq!(c.shoulds, vec!["rust"]);

        let c = parse(r#""new york" rust"#).into_clauses(BoolMode::And);
        assert_eq!(c.must_phrases, vec![vec!["new", "york"]]);
        assert!(c.should_phrases.is_empty());
        assert_eq!(c.musts, vec!["rust"]);
    }

    // ---- into_clauses (default-operator resolution) ----

    #[test]
    fn into_clauses_or_maps_bare_to_should() {
        let c = parse("+climate policy -spam").into_clauses(BoolMode::Or);
        assert_eq!(c.musts, vec!["climate"]);
        assert_eq!(c.shoulds, vec!["policy"]);
        assert_eq!(c.negatives, vec!["spam"]);
    }

    #[test]
    fn into_clauses_and_folds_bare_into_musts() {
        let c = parse("+climate policy -spam").into_clauses(BoolMode::And);
        assert_eq!(c.musts, vec!["climate", "policy"]);
        assert!(c.shoulds.is_empty());
        assert_eq!(c.negatives, vec!["spam"]);
    }

    #[test]
    fn into_clauses_legacy_shapes_unchanged() {
        // Sigil-less queries resolve exactly as the pre-clause model:
        // Or ⇒ all shoulds (union), And ⇒ all musts (intersection).
        let c = parse("rust async").into_clauses(BoolMode::Or);
        assert!(c.musts.is_empty());
        assert_eq!(c.shoulds, vec!["rust", "async"]);

        let c = parse("rust async").into_clauses(BoolMode::And);
        assert_eq!(c.musts, vec!["rust", "async"]);
        assert!(c.shoulds.is_empty());
    }

    #[test]
    fn parse_lowercase_tokens_borrow_the_query() {
        // Zero-copy contract: already-lowercase runs must not allocate.
        let p = parse("rust -python");
        assert!(matches!(p.positives[0], Cow::Borrowed(_)));
        assert!(matches!(p.negatives[0], Cow::Borrowed(_)));
    }

    #[test]
    fn parse_uppercase_token_is_the_only_copy() {
        let p = parse("rust -PYTHON");
        assert!(matches!(p.positives[0], Cow::Borrowed(_)));
        assert!(matches!(p.negatives[0], Cow::Owned(_)));
    }

    /// The `new` constructor plus the trait-object `tokenize_each`
    /// dispatch path (distinct from the inherent `tokenize_each_inline`
    /// the hot path uses). Mixed case and punctuation confirm the
    /// lowercasing + separator splitting.
    #[test]
    fn dyn_tokenize_each_lowercases_and_splits() {
        let tok = AsciiLowerTokenizer::new();
        let dynt: &dyn Tokenizer = &tok;
        let mut out = Vec::new();
        dynt.tokenize_each("Hello, World rust", &mut |s| out.push(s.to_string()));
        assert_eq!(out, vec!["hello", "world", "rust"]);
    }
}