wm_memory/

search.rs

//! Search engine — Tantivy full-text search.
//!
//! Provides BM25-scored full-text search over memory content.
//! Index is stored alongside the LMDB store in a separate directory.
//!
//! Recall-quality hygiene (see `docs/TANTIVY_RECALL_QUALITY_FIX.md`):
//! - Queries are stripped of common stopwords before parsing.
//! - Terms are only quoted when they contain reserved query syntax
//!   (plain terms — including hyphenated compounds — pass through so the
//!   tokenizer can split them into phrase matches).
//! - Content is sanitized at index time (binary/garbage content is skipped).
//! - Results are filtered by optional absolute and/or relative score floors,
//!   and output content is scrubbed of control characters.

use crate::MemoryId;
use serde::{Deserialize, Serialize};
use wm_core::{CoreError, Galaxy, Result};

use std::path::Path;
use std::sync::Mutex;
use std::sync::atomic::{AtomicU64, Ordering};
use tantivy::{
    Index, IndexReader, IndexWriter, ReloadPolicy, Term,
    collector::TopDocs,
    doc,
    query::QueryParser,
    schema::{
        Field, STORED, STRING, Schema, TantivyDocument, TextFieldIndexing, TextOptions, Value,
    },
};

/// Maximum content length (in chars) indexed into Tantivy.
pub const MAX_INDEX_CONTENT_LEN: usize = 8 * 1024;

/// Minimum printable-char ratio for content to be indexed (0.9 = max 10% garbage).
pub const MIN_PRINTABLE_RATIO: f32 = 0.9;

/// Common English stopwords stripped from queries before parsing.
///
/// Mirrors the client-side stopword list (Antigravity `wmMemory.ts`) so the
/// server and client agree on which tokens are meaningless for recall.
pub const STOPWORDS: &[&str] = &[
    "a",
    "about",
    "after",
    "again",
    "all",
    "also",
    "am",
    "an",
    "and",
    "any",
    "are",
    "as",
    "at",
    "be",
    "been",
    "being",
    "before",
    "between",
    "both",
    "but",
    "by",
    "can",
    "could",
    "did",
    "do",
    "does",
    "during",
    "each",
    "few",
    "for",
    "from",
    "further",
    "had",
    "has",
    "have",
    "he",
    "her",
    "here",
    "hers",
    "herself",
    "him",
    "himself",
    "his",
    "how",
    "i",
    "if",
    "in",
    "into",
    "is",
    "it",
    "its",
    "itself",
    "just",
    "me",
    "might",
    "more",
    "most",
    "my",
    "myself",
    "no",
    "nor",
    "not",
    "of",
    "off",
    "on",
    "once",
    "only",
    "or",
    "other",
    "our",
    "ours",
    "ourselves",
    "out",
    "over",
    "own",
    "same",
    "shall",
    "she",
    "should",
    "so",
    "some",
    "such",
    "than",
    "that",
    "the",
    "their",
    "theirs",
    "them",
    "themselves",
    "then",
    "there",
    "these",
    "they",
    "this",
    "those",
    "through",
    "to",
    "too",
    "under",
    "until",
    "up",
    "us",
    "very",
    "was",
    "we",
    "were",
    "what",
    "when",
    "where",
    "which",
    "while",
    "who",
    "whom",
    "why",
    "will",
    "with",
    "would",
    "you",
    "your",
    "yours",
    "yourself",
    "yourselves",
];

/// Search options controlling recall behavior.
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct SearchOptions {
    /// Maximum number of results.
    pub limit: usize,
    /// Optional galaxy filter (matches the stored galaxy string).
    pub galaxy: Option<Galaxy>,
    /// Absolute BM25 score floor; hits scoring below are dropped.
    pub min_score: Option<f32>,
    /// Relative floor: hits scoring below `top_score * ratio` are dropped
    /// (e.g. `0.05` keeps only hits within 5% of the top result).
    pub relative_floor: Option<f32>,
    /// Use OR semantics instead of conjunction (deprecated — OR is now the
    /// default; this flag is kept for API compatibility but does not change
    /// behavior).
    pub relaxed: bool,
}

impl Default for SearchOptions {
    fn default() -> Self {
        Self {
            limit: 20,
            galaxy: None,
            min_score: None,
            relative_floor: None,
            relaxed: false,
        }
    }
}

/// Search result item.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct SearchResult {
    /// Memory UUID (as string)
    pub memory_id: String,
    /// Galaxy name
    pub galaxy: String,
    /// Raw BM25 score
    pub score: f32,
    /// Score relative to the top hit (1.0 = top result, 0.0 = no results)
    pub normalized_score: f32,
    /// Content snippet (control characters scrubbed)
    pub content: String,
}

/// Tracked health of the Tantivy index relative to LMDB.
///
/// Because Tantivy indexing is best-effort (an indexing failure does not
/// roll back the LMDB write), the index can drift from the store. This
/// struct tracks successes and failures so `wm doctor` and `system.health`
/// can report degraded state instead of silently claiming healthy.
#[derive(Debug, Default)]
pub struct IndexHealth {
    /// Successful index/deindex operations since startup.
    pub successes: AtomicU64,
    /// Failed index/deindex operations since startup.
    pub failures: AtomicU64,
    /// Last error message (empty string if none).
    last_error: Mutex<String>,
}

impl IndexHealth {
    fn record_success(&self) {
        self.successes.fetch_add(1, Ordering::Relaxed);
    }

    fn record_failure(&self, err: &str) {
        self.failures.fetch_add(1, Ordering::Relaxed);
        if let Ok(mut guard) = self.last_error.lock() {
            *guard = err.to_string();
        }
    }

    /// Snapshot the health as a JSON value for tool output.
    #[must_use]
    pub fn snapshot(&self) -> serde_json::Value {
        let successes = self.successes.load(Ordering::Relaxed);
        let failures = self.failures.load(Ordering::Relaxed);
        let last_error = self
            .last_error
            .lock()
            .map(|g| g.clone())
            .unwrap_or_default();
        let degraded = failures > 0;
        serde_json::json!({
            "successes": successes,
            "failures": failures,
            "degraded": degraded,
            "last_error": if last_error.is_empty() { serde_json::Value::Null } else { serde_json::Value::String(last_error) },
        })
    }
}

/// Format the Tantivy writer-creation error with actionable lock context
/// (backlog B1: the bare `Lockfile: LockBusy` message named neither the
/// index path nor the likely holder, costing a debug session to isolate).
///
/// The lock case does not echo the raw error: Tantivy formats its lock
/// payload as a Debug-wrapped `Some("…")`, which reads like an internal
/// trace rather than an expected contention message (2026-09-15 audit).
fn format_writer_lock_error(err: &str, index_path: &Path) -> String {
    let is_lock = err.contains("ock") && (err.contains("Busy") || err.contains("lock"));
    if is_lock {
        format!(
            "search index lock busy — the index at {} is locked by another process. \
             A running `wm serve` or `wm daemon` on this store holds it; find it with \
             `pgrep -af wm` and stop it, or start this server with --readonly.",
            index_path.display()
        )
    } else {
        format!("Tantivy writer: {err}")
    }
}

/// The full-text search engine backed by Tantivy.
pub struct SearchEngine {
    index: Index,
    reader: IndexReader,
    writer: Mutex<Option<IndexWriter>>,
    field_id: Field,
    field_galaxy: Field,
    field_content: Field,
    field_tags: Field,
    field_timestamp: Field,
    /// Tracked index health — failures are recorded so callers can detect
    /// degraded state instead of silently reporting healthy.
    health: IndexHealth,
    /// True when the on-disk index had an incompatible schema and was
    /// replaced with a fresh empty index at open time. The old index was
    /// moved aside (`.schema-mismatch.<timestamp>` sibling); callers that
    /// own the canonical LMDB store should rebuild via
    /// [`crate::reindex::rebuild_index`].
    schema_migrated: bool,
}

impl SearchEngine {
    /// Build the Tantivy schema for memory indexing.
    fn build_schema() -> (Schema, Field, Field, Field, Field, Field) {
        let mut schema_builder = Schema::builder();
        let field_id = schema_builder.add_text_field("memory_id", STRING | STORED);
        let field_galaxy = schema_builder.add_text_field("galaxy", STRING | STORED);
        // Use en_stem tokenizer for content and tags so that morphological
        // variants match (e.g. "graduate" ↔ "graduated", "degree" ↔ "degrees").
        let stem_indexing = TextFieldIndexing::default()
            .set_tokenizer("en_stem")
            .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions);
        let stem_text = TextOptions::default()
            .set_indexing_options(stem_indexing.clone())
            .set_stored();
        let stem_tags = TextOptions::default().set_indexing_options(stem_indexing);
        let field_content = schema_builder.add_text_field("content", stem_text);
        let field_tags = schema_builder.add_text_field("tags", stem_tags);
        let field_timestamp = schema_builder.add_i64_field("timestamp", STORED);
        let schema = schema_builder.build();
        (
            schema,
            field_id,
            field_galaxy,
            field_content,
            field_tags,
            field_timestamp,
        )
    }

    /// Open (or create) the Tantivy index at `path`, migrating an
    /// incompatible schema when `writable`.
    ///
    /// The Tantivy index is a derived index over the canonical LMDB store.
    /// When the on-disk index was written by an older version with an
    /// incompatible schema, a writable open moves the old directory aside
    /// (`<name>.schema-mismatch.<millis>` sibling) and creates a fresh empty
    /// index — the caller should rebuild it from LMDB via
    /// [`crate::reindex::rebuild_index`]. A read-only open refuses to migrate
    /// and returns an error directing the user to `wm reindex`.
    fn open_index(path: &Path, schema: &Schema, writable: bool) -> Result<(Index, bool)> {
        let directory = tantivy::directory::MmapDirectory::open(path)
            .map_err(|e| CoreError::Memory(format!("Tantivy open directory: {e}")))?;
        if !writable {
            // `open_or_create` initializes an index in an existing but empty
            // directory. A preservation open must only accept an already
            // materialized index, never create its metadata or segments.
            let index = Index::open(directory).map_err(|e| {
                CoreError::Memory(format!(
                    "Tantivy readonly open-existing at {}: {e}",
                    path.display()
                ))
            })?;
            if index.schema() != *schema {
                return Err(CoreError::Memory(format!(
                    "Tantivy index at {} was created with an incompatible schema by an \
                     older version. Run 'wm reindex' (or start 'wm serve' without \
                     --readonly) to migrate and rebuild it from the canonical store.",
                    path.display()
                )));
            }
            return Ok((index, false));
        }
        match Index::open_or_create(directory, schema.clone()) {
            Ok(index) => Ok((index, false)),
            Err(tantivy::error::TantivyError::SchemaError(_)) => {
                let ts = std::time::SystemTime::now()
                    .duration_since(std::time::UNIX_EPOCH)
                    .map_or(0, |d| d.as_millis());
                let file_name = path
                    .file_name()
                    .and_then(|n| n.to_str())
                    .unwrap_or("tantivy");
                let backup = path.with_file_name(format!("{file_name}.schema-mismatch.{ts}"));
                std::fs::rename(path, &backup).map_err(|e| {
                    CoreError::Memory(format!(
                        "Tantivy schema migration — rename old index to {}: {e}",
                        backup.display()
                    ))
                })?;
                std::fs::create_dir_all(path).map_err(|e| {
                    CoreError::Memory(format!("Tantivy schema migration — create index dir: {e}"))
                })?;
                tracing::warn!(
                    "Tantivy index schema mismatch — old index moved to {}; creating a fresh \
                     index (rebuild from LMDB will follow)",
                    backup.display()
                );
                let directory = tantivy::directory::MmapDirectory::open(path)
                    .map_err(|e| CoreError::Memory(format!("Tantivy open directory: {e}")))?;
                let index = Index::open_or_create(directory, schema.clone())
                    .map_err(|e| CoreError::Memory(format!("Tantivy open_or_create: {e}")))?;
                Ok((index, true))
            }
            Err(e) => Err(CoreError::Memory(format!("Tantivy open_or_create: {e}"))),
        }
    }

    /// Create or open a search engine index at the given path.
    pub fn open(path: impl AsRef<Path>) -> Result<Self> {
        let path = path.as_ref();
        let (schema, field_id, field_galaxy, field_content, field_tags, field_timestamp) =
            Self::build_schema();

        let (index, schema_migrated) = Self::open_index(path, &schema, true)?;

        let reader = index
            .reader_builder()
            .reload_policy(ReloadPolicy::OnCommitWithDelay)
            .try_into()
            .map_err(|e| CoreError::Memory(format!("Tantivy reader: {e}")))?;

        let writer = index
            .writer(50_000_000)
            .map_err(|e| CoreError::Memory(format_writer_lock_error(&e.to_string(), path)))?;

        Ok(Self {
            index,
            reader,
            writer: Mutex::new(Some(writer)),
            field_id,
            field_galaxy,
            field_content,
            field_tags,
            field_timestamp,
            health: IndexHealth::default(),
            schema_migrated,
        })
    }

    /// Open the index in read-only mode: no writer is created, so no
    /// exclusive tantivy lock is taken. Multiple processes (e.g. Antigravity's
    /// proxy and an opencode MCP client) can share the store for searches;
    /// writes through this engine fail with a clear error.
    pub fn open_readonly(path: impl AsRef<Path>) -> Result<Self> {
        Self::open_readonly_with_disclosure(path, true)
    }

    /// [`open_readonly`] without the open-time write-visibility warning:
    /// for one-shot inspection processes (`wm doctor`) that exit before any
    /// later write could matter. The read-only server keeps the loud path —
    /// there the warning is the honest Backlog B2 disclosure.
    pub fn open_readonly_quiet(path: impl AsRef<Path>) -> Result<Self> {
        Self::open_readonly_with_disclosure(path, false)
    }

    fn open_readonly_with_disclosure(path: impl AsRef<Path>, loud: bool) -> Result<Self> {
        let path = path.as_ref();
        // Backlog B2: a read-only open never observes later writes — searches
        // miss fresh memories until restart. Say so at the moment it matters
        // (a long-lived reader); a one-shot CLI process exits first.
        if loud {
            tracing::warn!(
                "read-only search index opened at {} — it will not observe writes made \
                 after this point; restart the read-only server to pick up new memories",
                path.display()
            );
        } else {
            tracing::debug!(
                "read-only search index opened at {} (inspection process)",
                path.display()
            );
        }
        let (schema, field_id, field_galaxy, field_content, field_tags, field_timestamp) =
            Self::build_schema();
        let (index, schema_migrated) = Self::open_index(path, &schema, false)?;
        let reader = index
            .reader_builder()
            .reload_policy(ReloadPolicy::OnCommitWithDelay)
            .try_into()
            .map_err(|e| CoreError::Memory(format!("Tantivy reader: {e}")))?;
        Ok(Self {
            index,
            reader,
            writer: Mutex::new(None),
            field_id,
            field_galaxy,
            field_content,
            field_tags,
            field_timestamp,
            health: IndexHealth::default(),
            schema_migrated,
        })
    }

    /// True when the on-disk index had an incompatible schema and was
    /// replaced with a fresh empty index at open time. The old index was
    /// preserved as a `.schema-mismatch.<timestamp>` sibling directory.
    /// Callers that own the canonical LMDB store should rebuild via
    /// `reindex::rebuild_index` when this returns true.
    #[must_use]
    pub const fn schema_migrated(&self) -> bool {
        self.schema_migrated
    }

    /// Returns a snapshot of index health (success/failure counts, degraded
    /// flag, last error).
    #[must_use]
    pub const fn health(&self) -> &IndexHealth {
        &self.health
    }

    /// Count the number of indexed documents for a specific galaxy.
    ///
    /// Used by consistency checks to compare Tantivy doc counts against
    /// LMDB memory counts. Returns 0 if the index is empty or the galaxy
    /// has no documents.
    pub fn count_docs_in_galaxy(&self, galaxy: &str) -> Result<usize> {
        // Deterministic view: the OnCommitWithDelay background reloader can
        // lag a just-finished commit; drift classification must never run
        // against a stale reader (it would under-count and skip the heal).
        self.reader
            .reload()
            .map_err(|e| CoreError::Memory(format!("Tantivy reader reload: {e}")))?;
        let searcher = self.reader.searcher();
        let term = tantivy::Term::from_field_text(self.field_galaxy, galaxy);
        let query = tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic);
        let count = searcher
            .search(&query, &tantivy::collector::Count)
            .map_err(|e| CoreError::Memory(format!("Tantivy count_docs: {e}")))?;
        Ok(count)
    }

    /// Enumerate the memory IDs currently indexed for one galaxy.
    ///
    /// Used by the incremental drift heal to diff the index against LMDB
    /// without rebuilding whole galaxies. Bounded by the galaxy's own
    /// document count (a term query on the non-tokenized `galaxy` field, so
    /// the cost is one term seek + one stored-field fetch per hit).
    pub fn indexed_ids_in_galaxy(&self, galaxy: &str) -> Result<std::collections::HashSet<String>> {
        // Deterministic view: the OnCommitWithDelay background reloader can
        // lag a just-finished commit, and the drift heal must never diff
        // against a stale reader (it would re-index what it already did).
        self.reader
            .reload()
            .map_err(|e| CoreError::Memory(format!("Tantivy reader reload: {e}")))?;
        let searcher = self.reader.searcher();
        let term = tantivy::Term::from_field_text(self.field_galaxy, galaxy);
        let query = tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic);
        let count = self.count_docs_in_galaxy(galaxy)?;
        let hits: std::collections::HashSet<tantivy::DocAddress> = searcher
            .search(&query, &tantivy::collector::DocSetCollector)
            .map_err(|e| CoreError::Memory(format!("Tantivy indexed_ids: {e}")))?;
        let mut out = std::collections::HashSet::with_capacity(count);
        for addr in hits {
            let doc: TantivyDocument = searcher
                .doc(addr)
                .map_err(|e| CoreError::Memory(format!("Tantivy get doc: {e}")))?;
            if let Some(id) = doc.get_first(self.field_id).and_then(|v| v.as_str()) {
                out.insert(id.to_string());
            }
        }
        Ok(out)
    }

    /// True when the engine was opened read-only (no tantivy writer).
    pub fn is_readonly(&self) -> bool {
        self.writer.lock().map_or(true, |g| g.is_none())
    }

    /// Lock the shared writer for adding/removing documents.
    ///
    /// The writer is created at `open()` time and shared across all callers
    /// via a `Mutex`, preventing lock contention with Tantivy's single-writer model.
    /// In read-only mode this errors.
    pub fn writer(&self) -> Result<std::sync::MutexGuard<'_, Option<IndexWriter>>> {
        let guard = self
            .writer
            .lock()
            .map_err(|_| CoreError::Memory("Tantivy writer mutex poisoned".into()))?;
        if guard.is_none() {
            return Err(CoreError::Memory(
                "Tantivy writer unavailable: index opened read-only".into(),
            ));
        }
        Ok(guard)
    }

    /// Index a memory document.
    ///
    /// Content that is not clean text (binary garbage, low printable-char
    /// ratio, null bytes) is **skipped** at index time — no document is added
    /// and `Ok(())` is returned so callers can proceed. See
    /// [`sanitize_content_for_index`].
    pub fn add_document(
        &self,
        writer: &mut Option<IndexWriter>,
        memory_id: &str,
        galaxy: &str,
        content: &str,
        tags: &[String],
        timestamp: i64,
    ) -> Result<()> {
        let writer = writer.as_mut().ok_or_else(|| {
            CoreError::Memory("Tantivy writer unavailable: index opened read-only".into())
        })?;
        let Some(clean_content) = sanitize_content_for_index(content) else {
            tracing::debug!("Skipping index of memory {memory_id}: content failed sanitization");
            return Ok(());
        };
        let tags_str = tags.join(" ");
        let doc = doc!(
            self.field_id => memory_id,
            self.field_galaxy => galaxy,
            self.field_content => clean_content,
            self.field_tags => tags_str,
            self.field_timestamp => timestamp,
        );
        match writer.add_document(doc) {
            Ok(_) => {
                self.health.record_success();
                Ok(())
            }
            Err(e) => {
                let msg = format!("Tantivy add_document: {e}");
                self.health.record_failure(&msg);
                Err(CoreError::Memory(msg))
            }
        }
    }

    /// Index a complete `Memory` entry.
    pub fn index_memory(
        &self,
        writer: &mut Option<IndexWriter>,
        mem: &crate::memory::Memory,
    ) -> Result<()> {
        self.add_document(
            writer,
            &mem.metadata.id.to_string(),
            mem.metadata.galaxy.db_name(),
            &mem.content,
            &mem.metadata.tags,
            mem.metadata.created_at.timestamp(),
        )
    }

    /// Delete documents by memory ID.
    pub fn delete_document(&self, writer: &mut Option<IndexWriter>, memory_id: &str) -> Result<()> {
        let writer = writer.as_mut().ok_or_else(|| {
            CoreError::Memory("Tantivy writer unavailable: index opened read-only".into())
        })?;
        let term = tantivy::Term::from_field_text(self.field_id, memory_id);
        writer.delete_term(term);
        Ok(())
    }

    /// Delete every document belonging to a galaxy.
    ///
    /// Used by filtered reindexing so `--galaxy codex` removes only codex
    /// documents instead of wiping the entire index.
    pub fn delete_by_galaxy(&self, writer: &mut Option<IndexWriter>, galaxy: &str) -> Result<()> {
        let writer = writer.as_mut().ok_or_else(|| {
            CoreError::Memory("Tantivy writer unavailable: index opened read-only".into())
        })?;
        let term = tantivy::Term::from_field_text(self.field_galaxy, galaxy);
        writer.delete_term(term);
        Ok(())
    }

    /// Commit pending index changes and reload the reader.
    pub fn commit(&self, writer: &mut Option<IndexWriter>) -> Result<()> {
        let writer = writer.as_mut().ok_or_else(|| {
            CoreError::Memory("Tantivy writer unavailable: index opened read-only".into())
        })?;
        writer
            .commit()
            .map_err(|e| CoreError::Memory(format!("Tantivy commit: {e}")))?;
        self.reader
            .reload()
            .map_err(|e| CoreError::Memory(format!("Tantivy reload: {e}")))?;
        Ok(())
    }

    /// Search for memories matching the query text.
    /// Returns results sorted by BM25 score (descending).
    ///
    /// The query is stripped of stopwords and sanitized to prevent Tantivy
    /// query syntax injection.
    pub fn search(&self, query: &str, limit: usize) -> Result<Vec<SearchResult>> {
        let opts = SearchOptions {
            limit,
            ..SearchOptions::default()
        };
        self.search_opt(query, &opts)
    }

    /// Search for memories matching the query, optionally filtered by galaxy.
    ///
    /// The query is stripped of stopwords and sanitized to escape Tantivy
    /// special characters (+, -, *, "", field syntax, boolean operators) that
    /// could be used for query injection.
    pub fn search_in_galaxy(
        &self,
        query: &str,
        galaxy: Option<Galaxy>,
        limit: usize,
    ) -> Result<Vec<SearchResult>> {
        let opts = SearchOptions {
            limit,
            galaxy,
            ..SearchOptions::default()
        };
        self.search_opt(query, &opts)
    }

    /// Search with full recall-quality options (stopword stripping, score
    /// thresholds, token-coverage filtering, galaxy filter).
    ///
    /// Pipeline:
    /// 1. `strip_stopwords` — common English stopwords are removed.
    /// 2. `sanitize_tantivy_query` — reserved query syntax is neutralized;
    ///    plain terms (incl. hyphenated compounds) pass through so the
    ///    tokenizer can split them into phrase matches.
    /// 3. OR query across `content` + `tags` (broader recall than
    ///    conjunction, filtered by token-coverage in step 5).
    /// 4. Hits below `min_score` (absolute) or `relative_floor * top_score`
    ///    are dropped.
    /// 5. Token-coverage floor: for queries with ≥ 3 terms, at least 2
    ///    must appear in the content (stemming-aware).  Documents that
    ///    pass the floor receive a coverage-ratio score boost.
    /// 6. Output content is scrubbed of control characters.
    pub fn search_opt(&self, query: &str, opts: &SearchOptions) -> Result<Vec<SearchResult>> {
        // Tantivy's TopDocs panics on limit 0 ("Limit must be greater than
        // 0") — inside a server that panic kills the process (2026-09-19
        // review: memory.search with limit 0 exited 101). Reject before the
        // collector is built; the tool boundary reports the caller error,
        // this guard protects every other path into the engine.
        if opts.limit == 0 {
            return Err(CoreError::InvalidArgs(
                "search limit must be >= 1, got: 0".into(),
            ));
        }
        let stripped = strip_stopwords(query);
        let sanitized = sanitize_tantivy_query(&stripped);
        if sanitized.trim().is_empty() {
            return Ok(Vec::new());
        }

        let searcher = self.reader.searcher();

        // Always use OR semantics.  The token-coverage floor below filters
        // single-term noise that OR would otherwise let through.
        let query_parser =
            QueryParser::for_index(&self.index, vec![self.field_content, self.field_tags]);

        let parsed = parse_query_with_fallback(&query_parser, &sanitized);

        let collector = TopDocs::with_limit(opts.limit).order_by_score();

        let top_docs = searcher
            .search(&parsed, &collector)
            .map_err(|e| CoreError::Memory(format!("Tantivy search: {e}")))?;

        let top_score = top_docs.first().map_or(0.0, |(score, _)| *score);
        let absolute_floor = opts.min_score.unwrap_or(f32::MIN);
        let relative_floor = opts
            .relative_floor
            .map_or(f32::MIN, |ratio| top_score * ratio);

        // Token-coverage floor: with OR semantics a document matching any
        // single common term would otherwise qualify.  For queries with
        // ≥ 3 terms, require at least 2 to appear in the content — but count
        // only tokens that exist in the index at all.  A question word the
        // corpus never contains ("many" in "how many capabilities are
        // there?") must not raise the floor and filter every valid hit.
        let query_tokens = query_stem_tokens(&stripped);
        let present_tokens = query_tokens
            .iter()
            .filter(|token| {
                searcher
                    .doc_freq(&Term::from_field_text(self.field_content, token))
                    .unwrap_or(0)
                    > 0
            })
            .count();
        let coverage_floor = if query_tokens.len() >= 3 && present_tokens >= 2 {
            2
        } else {
            1
        };

        let mut results = Vec::new();
        for (score, doc_address) in top_docs {
            // Score floors: reject weak matches before touching the document.
            if score < absolute_floor || score < relative_floor {
                continue;
            }

            let doc: TantivyDocument = searcher
                .doc(doc_address)
                .map_err(|e| CoreError::Memory(format!("Tantivy get doc: {e}")))?;

            let memory_id = doc
                .get_first(self.field_id)
                .and_then(|v| v.as_str())
                .unwrap_or("")
                .to_string();

            let doc_galaxy = doc
                .get_first(self.field_galaxy)
                .and_then(|v| v.as_str())
                .unwrap_or("")
                .to_string();

            if let Some(g) = opts.galaxy {
                if doc_galaxy != g.db_name() {
                    continue;
                }
            }

            let content = doc
                .get_first(self.field_content)
                .and_then(|v| v.as_str())
                .unwrap_or("")
                .to_string();

            if coverage_floor > 1 {
                let hits = count_token_hits(&content, &stripped);
                if hits < coverage_floor {
                    continue;
                }
            }

            // Coverage-ratio boost: documents covering more query tokens
            // are more relevant.  Boost = 1 + 0.1 * (hits / total).
            let boosted_score = if query_tokens.is_empty() {
                score
            } else {
                let hits = count_token_hits(&content, &stripped);
                let ratio = hits as f32 / query_tokens.len() as f32;
                score * 0.1f32.mul_add(ratio, 1.0)
            };

            results.push(SearchResult {
                memory_id,
                galaxy: doc_galaxy,
                score: boosted_score,
                normalized_score: 0.0, // set after re-sort
                content: scrub_text(&content),
            });
        }

        // Re-sort by boosted score (coverage boost may have re-ordered).
        results.sort_by(|a, b| {
            b.score
                .partial_cmp(&a.score)
                .unwrap_or(std::cmp::Ordering::Equal)
        });

        // Normalize relative to the top boosted score.
        let top_boosted = results.first().map_or(0.0, |r| r.score);
        for r in &mut results {
            r.normalized_score = if top_boosted > 0.0 {
                r.score / top_boosted
            } else {
                0.0
            };
        }

        Ok(results)
    }

    /// Search and return memory IDs only (for integration with `MemoryStore`).
    pub fn search_ids(&self, query: &str, limit: usize) -> Result<Vec<MemoryId>> {
        let results = self.search(query, limit)?;
        Ok(results
            .into_iter()
            .filter_map(|r| uuid::Uuid::parse_str(&r.memory_id).ok())
            .collect())
    }
}

/// Parse a sanitized query, falling back to lenient parsing when the strict
/// parser rejects it.
///
/// [`sanitize_tantivy_query`] neutralizes known syntax, but the parser can
/// still reject input it does not anticipate (for example a term whose
/// quoting produces a dangling escape). Lenient parsing turns unparseable
/// fragments into match-nothing clauses, so a malformed query degrades to a
/// partial search instead of failing the request.
fn parse_query_with_fallback(parser: &QueryParser, query: &str) -> Box<dyn tantivy::query::Query> {
    match parser.parse_query(query) {
        Ok(parsed) => parsed,
        Err(_) => parser.parse_query_lenient(query).0,
    }
}

/// Sanitize a user-provided query string for Tantivy's query parser.
///
/// Tantivy's query parser supports special syntax that could be abused:
/// - `*` wildcard matches all terms (DoS)
/// - `+`, `-`, `NOT`, `OR`, `AND` boolean operators
/// - `"phrase"` exact phrase queries
/// - `field:value` field-scoped queries
/// - `(`, `)` grouping
/// - `\` escape character
/// - `:` field separator
///
/// Terms are only wrapped in double quotes when they contain reserved syntax
/// (or are uppercase boolean operators). Plain terms — including hyphenated
/// compounds like `antigravity-project-test` — pass through unquoted so the
/// tokenizer can split them into phrase matches. Terms without any
/// alphanumeric characters are dropped entirely.
#[must_use]
pub fn sanitize_tantivy_query(input: &str) -> String {
    // If empty, return as-is
    if input.trim().is_empty() {
        return String::new();
    }

    input
        .split_whitespace()
        .filter(|term| term.chars().any(char::is_alphanumeric))
        .map(|term| {
            if term_needs_quoting(term) {
                // Escape backslashes first, then embedded double quotes. A
                // trailing backslash would otherwise escape the closing quote
                // and produce an unterminated phrase (a parse error).
                let escaped = term.replace('\\', "\\\\").replace('"', "\\\"");
                format!("\"{escaped}\"")
            } else {
                term.to_string()
            }
        })
        .collect::<Vec<_>>()
        .join(" ")
}

/// Whether a query term needs quoting to neutralize Tantivy query syntax.
#[must_use]
fn term_needs_quoting(term: &str) -> bool {
    if term.starts_with('+') || term.starts_with('-') || term.starts_with('!') {
        return true;
    }
    if term == "AND" || term == "OR" || term == "NOT" {
        return true;
    }
    if term.contains("&&") || term.contains("||") {
        return true;
    }
    term.chars().any(|c| {
        matches!(
            c,
            '(' | ')' | '{' | '}' | '[' | ']' | '^' | '"' | '~' | '*' | '?' | ':' | '\\' | '/'
        )
    })
}

/// Strip common English stopwords from a query string.
///
/// Tokens are compared case-insensitively against [`STOPWORDS`] after
/// trimming surrounding punctuation, so a question ending in `?`
/// ("... are there?") strips exactly like the same words without it.
#[must_use]
pub fn strip_stopwords(query: &str) -> String {
    query
        .split_whitespace()
        .filter(|term| {
            let normalized: String = term
                .chars()
                .filter(|character| character.is_alphanumeric())
                .collect::<String>()
                .to_lowercase();
            !normalized.is_empty() && !STOPWORDS.contains(&normalized.as_str())
        })
        .collect::<Vec<_>>()
        .join(" ")
}

/// Unique lowercase stemmed tokens of a stopword-stripped query.
/// Uses [`simple_stem`] so that coverage matching aligns with the en_stem
/// tokenizer used at index time.
#[must_use]
fn query_stem_tokens(stripped_query: &str) -> Vec<String> {
    stem_tokens(stripped_query)
}

/// Normalize text into the same punctuation-delimited tokens on both sides
/// of the coverage comparison. This keeps possessives and hyphenated terms
/// from becoming query-only tokens or standalone one-character fragments.
#[must_use]
fn stem_tokens(text: &str) -> Vec<String> {
    let mut tokens: Vec<String> = Vec::new();
    for term in text
        .split(|c: char| !c.is_alphanumeric())
        .filter(|term| term.len() > 1)
    {
        let stemmed = simple_stem(&term.to_lowercase());
        if !tokens.contains(&stemmed) {
            tokens.push(stemmed);
        }
    }
    tokens
}

/// Lightweight suffix-stripping stemmer that approximates the Porter stemmer
/// used by Tantivy's `en_stem` tokenizer.  Handles the common English
/// inflections (-s, -es, -ed, -ing, -ly, -ies, -ied) without pulling in a
/// full stemming crate.  This is intentionally conservative — false
/// negatives (under-stemming) only make coverage stricter, never looser.
#[must_use]
fn simple_stem(word: &str) -> String {
    if word.len() <= 3 {
        return word.to_string();
    }
    // Order matters: check longer suffixes first.
    for suffix in ["ies", "ied", "ing", "edly", "ed", "ly", "es", "s"] {
        if let Some(stem) = word.strip_suffix(suffix) {
            // "ies" / "ied" → restore "y" (stories → story, carried → carry)
            if suffix == "ies" || suffix == "ied" {
                return format!("{stem}y");
            }
            // Don't produce a 1-char stem ("is" → "i")
            if stem.len() >= 2 {
                return stem.to_string();
            }
        }
    }
    word.to_string()
}

/// Count how many query tokens (after stemming) appear as whole words in the
/// content.  Uses [`simple_stem`] on both sides so that "graduate" matches
/// "graduated", mirroring the en_stem tokenizer used at index time.
#[must_use]
fn count_token_hits(content: &str, stripped_query: &str) -> usize {
    let query_tokens = query_stem_tokens(stripped_query);
    if query_tokens.is_empty() {
        return 0;
    }
    let content_stems: std::collections::HashSet<String> =
        stem_tokens(content).into_iter().collect();
    query_tokens
        .iter()
        .filter(|t| content_stems.contains(*t))
        .count()
}

/// Printable-character ratio used by the index/admission gate.
///
/// Tab, newline, and carriage return are **formatting whitespace, not
/// debris** — they count as printable, aligned with the byte-level ingest
/// gate (`wm_mcp::ingest::binary_content`), which already treats those
/// three bytes as printable. Any other Unicode control character counts
/// against the ratio. Empty content returns 1.0 (vacuously clean); callers
/// reject emptiness separately.
///
/// 2026-09-19 benchmark finding: the old definition counted line breaks as
/// unprintable, so code/formatting-heavy memories (hex-color lists, HTML
/// and jQuery snippets) failed admission even though they are exactly the
/// content a coding-agent memory exists to keep.
#[must_use]
pub fn printable_ratio(content: &str) -> f32 {
    let total = content.chars().count();
    if total == 0 {
        return 1.0;
    }
    let printable = content
        .chars()
        .filter(|c| !c.is_control() || matches!(c, '\t' | '\n' | '\r'))
        .count();
    printable as f32 / total as f32
}

/// Prepare content for indexing.
///
/// Returns `None` when the content is not clean text and must be skipped:
/// - empty / whitespace-only content
/// - contains a null byte (binary serialization artifact)
/// - printable-char ratio below [`MIN_PRINTABLE_RATIO`] (tab/newline/CR
///   count as printable — see [`printable_ratio`])
///
/// Otherwise returns the content scrubbed of control characters and capped
/// at [`MAX_INDEX_CONTENT_LEN`] chars.
#[must_use]
pub fn sanitize_content_for_index(content: &str) -> Option<String> {
    if content.trim().is_empty() {
        return None;
    }
    if content.as_bytes().contains(&0) {
        return None;
    }
    if printable_ratio(content) < MIN_PRINTABLE_RATIO {
        return None;
    }

    let cleaned = scrub_text(content);
    let capped: String = cleaned.chars().take(MAX_INDEX_CONTENT_LEN).collect();
    if capped.trim().is_empty() {
        None
    } else {
        Some(capped)
    }
}

/// Scrub text for output: replace control characters (except newline, tab,
/// carriage return) with a space, and cap the length at
/// [`MAX_INDEX_CONTENT_LEN`].
#[must_use]
pub fn scrub_text(content: &str) -> String {
    let mut out = String::with_capacity(content.len().min(MAX_INDEX_CONTENT_LEN));
    for c in content.chars().take(MAX_INDEX_CONTENT_LEN) {
        if c.is_control() && c != '\n' && c != '\t' && c != '\r' {
            out.push(' ');
        } else {
            out.push(c);
        }
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::tempdir;

    fn open_engine() -> (tempfile::TempDir, SearchEngine) {
        let tmp = tempdir().unwrap();
        let engine = SearchEngine::open(tmp.path()).unwrap();
        (tmp, engine)
    }

    /// 2026-09-19 review: limit 0 reached Tantivy's `TopDocs` and panicked
    /// the server process (exit 101). The engine must return a caller error
    /// before the collector is built — this guard covers every path in.
    #[test]
    fn search_rejects_zero_limit_without_panicking() {
        let (_tmp, engine) = open_engine();
        let err = engine.search("anything", 0).unwrap_err();
        assert!(err.to_string().contains("limit"), "{err}");
        let err = engine
            .search_in_galaxy("anything", Some(Galaxy::Codex), 0)
            .unwrap_err();
        assert!(err.to_string().contains("limit"), "{err}");
        let opts = SearchOptions {
            limit: 0,
            ..SearchOptions::default()
        };
        assert!(engine.search_opt("anything", &opts).is_err());
    }

    /// Write a legacy one-field index into `dir`, simulating a store created
    /// by an older WhiteMagic version with a different Tantivy schema.
    fn write_incompatible_index(dir: &Path) {
        std::fs::create_dir_all(dir).unwrap();
        let mut builder = Schema::builder();
        builder.add_text_field("legacy", STRING | STORED);
        let schema = builder.build();
        let directory = tantivy::directory::MmapDirectory::open(dir).unwrap();
        Index::open_or_create(directory, schema).unwrap();
    }

    #[test]
    fn open_migrates_incompatible_schema() {
        let tmp = tempdir().unwrap();
        let dir = tmp.path().join("tantivy");
        write_incompatible_index(&dir);

        let engine = SearchEngine::open(&dir).unwrap();
        assert!(
            engine.schema_migrated(),
            "incompatible schema must trigger migration"
        );

        // The old index must be preserved as a .schema-mismatch sibling.
        let backups: Vec<_> = std::fs::read_dir(tmp.path())
            .unwrap()
            .filter_map(std::result::Result::ok)
            .filter(|e| e.file_name().to_string_lossy().contains("schema-mismatch"))
            .collect();
        assert_eq!(backups.len(), 1, "old index must be backed up exactly once");

        // The fresh index must be writable and searchable.
        let mut writer = engine.writer().unwrap();
        engine
            .add_document(
                &mut writer,
                "33333333-3333-3333-3333-333333333333",
                "codex",
                "fresh index after migration",
                &[],
                1700000000,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();
        let results = engine.search("fresh index", 10).unwrap();
        assert_eq!(results.len(), 1);
    }

    #[test]
    fn writer_lock_error_names_path_and_hint() {
        // B1: LockBusy used to surface bare ("Failed to acquire Lockfile:
        // LockBusy") with no path and no hint — a stray `wm serve` cost a
        // debug session to find.
        let err = format_writer_lock_error(
            "Failed to acquire Lockfile: LockBusy. Some(\"...\")",
            Path::new("/store/x/tantivy"),
        );
        assert!(
            err.contains("/store/x/tantivy"),
            "must name the index path: {err}"
        );
        assert!(
            err.contains("pgrep -af wm"),
            "must include the diagnostic hint: {err}"
        );
        assert!(
            err.contains("--readonly"),
            "must offer the readonly alternative: {err}"
        );

        // Non-lock errors pass through unchanged.
        let other = format_writer_lock_error("disk full", Path::new("/s/t"));
        assert!(other.starts_with("Tantivy writer: disk full"));
        assert!(!other.contains("pgrep"));
    }

    #[test]
    fn open_readonly_rejects_incompatible_schema() {
        let tmp = tempdir().unwrap();
        let dir = tmp.path().join("tantivy");
        write_incompatible_index(&dir);

        let err = match SearchEngine::open_readonly(&dir) {
            Ok(_) => panic!("read-only open must reject an incompatible schema"),
            Err(e) => e,
        };
        assert!(
            format!("{err}").contains("wm reindex"),
            "read-only mismatch must point at wm reindex, got: {err}"
        );

        // Nothing was moved: the incompatible index is still in place.
        let siblings: Vec<_> = std::fs::read_dir(tmp.path())
            .unwrap()
            .filter_map(std::result::Result::ok)
            .filter(|e| e.file_name().to_string_lossy().contains("schema-mismatch"))
            .collect();
        assert!(siblings.is_empty(), "read-only open must not migrate");
    }

    #[test]
    fn open_readonly_rejects_existing_empty_directory_without_creating_files() {
        let tmp = tempdir().unwrap();
        let dir = tmp.path().join("tantivy");
        std::fs::create_dir_all(&dir).unwrap();

        let err = match SearchEngine::open_readonly(&dir) {
            Ok(_) => panic!("readonly open unexpectedly initialized an empty index"),
            Err(err) => err,
        };
        assert!(format!("{err}").contains("readonly open-existing"));
        assert!(
            std::fs::read_dir(&dir).unwrap().next().is_none(),
            "readonly open must not materialize Tantivy metadata or segments"
        );
    }

    #[test]
    fn reopen_matching_schema_not_migrated() {
        let tmp = tempdir().unwrap();
        let dir = tmp.path().join("tantivy");
        std::fs::create_dir_all(&dir).unwrap();

        let first = SearchEngine::open(&dir).unwrap();
        assert!(!first.schema_migrated());
        drop(first); // release the tantivy writer lock before reopening

        let second = SearchEngine::open(&dir).unwrap();
        assert!(
            !second.schema_migrated(),
            "matching schema must not migrate"
        );
        drop(second);

        let third = SearchEngine::open_readonly(&dir).unwrap();
        assert!(!third.schema_migrated());
    }

    #[test]
    fn index_and_search_basic() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "The Rust programming language is fast and safe",
                &["rust".into(), "programming".into()],
                1700000000,
            )
            .unwrap();
        engine
            .add_document(
                &mut writer,
                "22222222-2222-2222-2222-222222222222",
                "codex",
                "Python is great for data science",
                &["python".into(), "data".into()],
                1700000001,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        let results = engine.search("rust", 10).unwrap();
        assert!(!results.is_empty());
        assert_eq!(results[0].memory_id, "11111111-1111-1111-1111-111111111111");
    }

    #[test]
    fn search_by_tag() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "memory about systems",
                &["rust".into()],
                1700000000,
            )
            .unwrap();
        engine
            .add_document(
                &mut writer,
                "22222222-2222-2222-2222-222222222222",
                "codex",
                "memory about cooking",
                &["food".into()],
                1700000001,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        let results = engine.search("rust", 10).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].memory_id, "11111111-1111-1111-1111-111111111111");
    }

    #[test]
    fn search_filtered_by_galaxy() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "important knowledge",
                &[],
                1700000000,
            )
            .unwrap();
        engine
            .add_document(
                &mut writer,
                "22222222-2222-2222-2222-222222222222",
                "research",
                "important findings",
                &[],
                1700000001,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        let results = engine
            .search_in_galaxy("important", Some(Galaxy::Codex), 10)
            .unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].galaxy, "codex");
    }

    #[test]
    fn delete_document_from_index() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "deletable content",
                &[],
                1700000000,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        let results = engine.search("deletable", 10).unwrap();
        assert_eq!(results.len(), 1);

        engine
            .delete_document(&mut writer, "11111111-1111-1111-1111-111111111111")
            .unwrap();
        engine.commit(&mut writer).unwrap();

        let results = engine.search("deletable", 10).unwrap();
        assert_eq!(results.len(), 0);
    }

    #[test]
    fn search_empty_index() {
        let (_tmp, engine) = open_engine();
        let results = engine.search("anything", 10).unwrap();
        assert!(results.is_empty());
    }

    #[test]
    fn search_ids_returns_uuids() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "unique content about rust",
                &[],
                1700000000,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        let ids = engine.search_ids("rust", 10).unwrap();
        assert_eq!(ids.len(), 1);
        assert_eq!(
            ids[0],
            uuid::Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap()
        );
    }

    // ── Tantivy query injection tests ───────────────────────────────

    #[test]
    fn sanitize_leaves_plain_terms_unquoted() {
        let result = sanitize_tantivy_query("hello world");
        assert_eq!(result, "hello world");
    }

    #[test]
    fn sanitize_drops_punct_only_terms() {
        let result = sanitize_tantivy_query("*");
        assert_eq!(result, "");
        // Should not match all documents when parsed
    }

    #[test]
    fn sanitize_escapes_boolean_operators() {
        let result = sanitize_tantivy_query("NOT secret");
        assert_eq!(result, "\"NOT\" secret");
    }

    #[test]
    fn sanitize_escapes_field_syntax() {
        let result = sanitize_tantivy_query("content:secret");
        assert_eq!(result, "\"content:secret\"");
    }

    #[test]
    fn sanitize_escapes_quotes() {
        let result = sanitize_tantivy_query("test\"injection");
        assert!(
            result.contains("\\\""),
            "embedded quotes should be escaped: {result}"
        );
    }

    #[test]
    fn sanitize_escapes_trailing_backslash_token() {
        // A term ending in a backslash used to become "abc\" — the dangling
        // escape swallowed the closing quote and failed the query parser.
        let result = sanitize_tantivy_query("C:\\Users\\temp\\");
        assert_eq!(
            result, "\"C:\\\\Users\\\\temp\\\\\"",
            "backslashes must be doubled inside quoted terms"
        );
    }

    #[test]
    fn lenient_fallback_never_fails_on_malformed_input() {
        let (_tmp, engine) = open_engine();
        let parser =
            QueryParser::for_index(&engine.index, vec![engine.field_content, engine.field_tags]);
        let searcher = engine.reader.searcher();
        let collector = TopDocs::with_limit(1).order_by_score();
        for malformed in ["\"unterminated", "field:(\"", "\\", "AND NOT OR"] {
            let parsed = parse_query_with_fallback(&parser, malformed);
            searcher
                .search(&parsed, &collector)
                .unwrap_or_else(|e| panic!("lenient query {malformed:?} must execute: {e}"));
        }
    }

    #[test]
    fn sanitize_empty_returns_empty() {
        assert_eq!(sanitize_tantivy_query(""), "");
        assert_eq!(sanitize_tantivy_query("   "), "");
    }

    #[test]
    fn sanitize_preserves_alphanumeric() {
        let result = sanitize_tantivy_query("rust programming 2024");
        assert_eq!(result, "rust programming 2024");
    }

    #[test]
    fn sanitize_preserves_hyphenated_compounds() {
        let result = sanitize_tantivy_query("antigravity antigravity-project-test");
        assert_eq!(result, "antigravity antigravity-project-test");
    }

    // ── Stopword tests ──────────────────────────────────────────────

    #[test]
    fn strip_stopwords_removes_common_words() {
        assert_eq!(
            strip_stopwords("smoke test from wmClient"),
            "smoke test wmClient"
        );
        assert_eq!(strip_stopwords("the from and or"), "");
        assert_eq!(strip_stopwords("Rust ownership"), "Rust ownership");
        assert_eq!(strip_stopwords(""), "");
    }

    #[test]
    fn strip_stopwords_is_case_insensitive() {
        assert_eq!(strip_stopwords("FROM The And"), "");
    }

    // ── Index-time sanitization tests ───────────────────────────────

    #[test]
    fn sanitize_content_skips_null_bytes() {
        let content = "binary\x00garbage\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00";
        assert!(sanitize_content_for_index(content).is_none());
    }

    #[test]
    fn sanitize_content_skips_low_printable_ratio() {
        // 5 control chars out of 11 → ratio 0.55 < 0.9 → skip
        let content = "\u{01}\u{02}\u{03}\u{04}\u{05}hello";
        assert!(sanitize_content_for_index(content).is_none());
    }

    #[test]
    fn sanitize_content_accepts_code_and_formatting_heavy_text() {
        // 2026-09-19 benchmark regression: code/formatting-heavy turns were
        // rejected because every line break counted as unprintable. Tab,
        // newline, and CR are formatting whitespace — they must not fail
        // admission (the byte-level ingest gate already treats them as
        // printable bytes).
        let hex_list = "Casper\n#ACBFCD\n\nPickled Bluewood\n#324558\n\nComet\n#545B70\n";
        assert!(sanitize_content_for_index(hex_list).is_some());
        let html = "Sure, here's how:\n```html\n<!DOCTYPE html>\n<html>\n  <body>\n \n \n  </body>\n</html>\n```";
        assert!(sanitize_content_for_index(html).is_some());
        // Genuine binary control chars still fail the ratio.
        let binary = "\u{01}\u{02}\u{03}\u{04}\u{05}hello";
        assert!(sanitize_content_for_index(binary).is_none());
    }

    #[test]
    fn printable_ratio_treats_line_breaks_as_printable() {
        assert!((printable_ratio("a\nb\tc\rd") - 1.0).abs() < f32::EPSILON);
        assert!(printable_ratio("\u{01}\u{02}\u{03}\u{04}\u{05}hello") < 0.9);
        assert!((printable_ratio("") - 1.0).abs() < f32::EPSILON);
    }

    #[test]
    fn sanitize_content_scrubs_and_caps() {
        // A stray control char does not disqualify clean text — it is scrubbed.
        let content = "clean text\u{01}with one control char";
        let cleaned = sanitize_content_for_index(content).unwrap();
        assert!(!cleaned.contains('\u{01}'));
        assert!(cleaned.starts_with("clean text with one control char"));

        let long = "a".repeat(MAX_INDEX_CONTENT_LEN + 1000);
        let capped = sanitize_content_for_index(&long).unwrap();
        assert_eq!(capped.chars().count(), MAX_INDEX_CONTENT_LEN);
    }

    #[test]
    fn sanitize_content_skips_empty() {
        assert!(sanitize_content_for_index("").is_none());
        assert!(sanitize_content_for_index("   ").is_none());
    }

    #[test]
    fn scrub_text_replaces_control_chars() {
        let result = scrub_text("a\u{01}b\nc\td\u{7f}e");
        assert_eq!(result, "a b\nc\td e");
    }

    #[test]
    fn add_document_skips_binary_content() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "\u{00}\u{01}\u{02}raw serialized bytes",
                &[],
                1700000000,
            )
            .unwrap();
        engine
            .add_document(
                &mut writer,
                "22222222-2222-2222-2222-222222222222",
                "codex",
                "clean searchable text",
                &[],
                1700000001,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        // The binary doc must not be searchable; the clean one must be.
        let results = engine.search("serialized", 10).unwrap();
        assert!(results.is_empty(), "binary content must not be indexed");

        let results = engine.search("clean", 10).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].memory_id, "22222222-2222-2222-2222-222222222222");
    }

    // ── Score-threshold tests ───────────────────────────────────────

    fn index_alpha_pair(engine: &SearchEngine) {
        let mut writer = engine.writer().unwrap();
        // Two docs both containing "alpha"; the short one scores higher
        // (BM25 length normalization).
        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "alpha",
                &[],
                1700000000,
            )
            .unwrap();
        let filler = format!("alpha {}", "zzz ".repeat(400));
        engine
            .add_document(
                &mut writer,
                "22222222-2222-2222-2222-222222222222",
                "codex",
                &filler,
                &[],
                1700000001,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();
    }

    #[test]
    fn strip_stopwords_ignores_trailing_punctuation() {
        assert_eq!(
            strip_stopwords("how many capabilities are there?"),
            "many capabilities"
        );
    }

    /// Regression: a natural-language question whose words are mostly absent
    /// from the corpus returned zero results. The trailing "?" kept "there?"
    /// from being recognized as a stopword, which pushed the query over the
    /// token-coverage floor; the floor now counts only index-present tokens.
    #[test]
    fn question_with_absent_token_still_matches() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();
        engine
            .add_document(
                &mut writer,
                "33333333-3333-3333-3333-333333333333",
                "codex",
                "unique zebra content",
                &[],
                1700000002,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        // "many", "stripes", and "exist" are absent from the corpus; the old
        // all-query-token floor (2 of 4) filtered the only valid hit.
        let results = engine.search("how many zebra stripes exist?", 5).unwrap();
        assert_eq!(
            results.len(),
            1,
            "the single valid hit must survive the coverage floor"
        );
        assert_eq!(results[0].memory_id, "33333333-3333-3333-3333-333333333333");
    }

    #[test]
    fn search_absolute_min_score_filters_weak_matches() {
        let (_tmp, engine) = open_engine();
        index_alpha_pair(&engine);

        let base = engine.search("alpha", 10).unwrap();
        assert_eq!(base.len(), 2);
        let (hi, lo) = if base[0].score >= base[1].score {
            (base[0].score, base[1].score)
        } else {
            (base[1].score, base[0].score)
        };
        assert!(
            hi > lo,
            "short doc should outscore long doc (hi={hi}, lo={lo})"
        );
        let mid = f32::midpoint(hi, lo);

        let opts = SearchOptions {
            limit: 10,
            min_score: Some(mid),
            ..SearchOptions::default()
        };
        let filtered = engine.search_opt("alpha", &opts).unwrap();
        assert_eq!(filtered.len(), 1);
        assert!((filtered[0].score - hi).abs() < 1e-3);
    }

    #[test]
    fn search_relative_floor_filters_weak_matches() {
        let (_tmp, engine) = open_engine();
        index_alpha_pair(&engine);

        let opts = SearchOptions {
            limit: 10,
            relative_floor: Some(0.5),
            ..SearchOptions::default()
        };
        let filtered = engine.search_opt("alpha", &opts).unwrap();
        assert_eq!(filtered.len(), 1, "weak match must fall below 50% of top");
        assert_eq!(
            filtered[0].memory_id,
            "11111111-1111-1111-1111-111111111111"
        );
        assert!((filtered[0].normalized_score - 1.0).abs() < 1e-3);
    }

    #[test]
    fn search_all_results_normalized() {
        let (_tmp, engine) = open_engine();
        index_alpha_pair(&engine);

        let results = engine.search("alpha", 10).unwrap();
        assert_eq!(results.len(), 2);
        assert!((results[0].normalized_score - 1.0).abs() < 1e-3);
        for r in &results[1..] {
            assert!(r.normalized_score <= 1.0);
            assert!(r.normalized_score > 0.0);
        }
    }

    #[test]
    fn search_stemming_matches_morphological_variants() {
        // The en_stem tokenizer should match morphological variants:
        // "graduate" (query) ↔ "graduated" (indexed content).
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "I graduated with a degree in Business Administration",
                &[],
                1700000000,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        // Query with present tense "graduate" should match past tense "graduated"
        let results = engine.search("graduate", 10).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].memory_id, "11111111-1111-1111-1111-111111111111");

        // Query with "degrees" should match "degree"
        let results = engine.search("degrees", 10).unwrap();
        assert_eq!(results.len(), 1);
    }

    #[test]
    fn search_incident_query_returns_only_relevant() {
        // Mirrors the 2026-08-11 incident: `memory.hybrid_recall`
        // query "smoke test from wmClient" must NOT return unrelated memories.
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        let smoke_id = "11111111-1111-1111-1111-111111111111";
        engine
            .add_document(
                &mut writer,
                smoke_id,
                "codex",
                "smoke test from wmClient: verify recall works",
                &[],
                1700000000,
            )
            .unwrap();
        let unrelated = [
            "NES Evolution and Impact: a history of the console wars",
            "Insights on The Gateless Gate: koans and zen practice",
            "What the tweet is really saying: a thread analysis",
        ];
        for (i, content) in (1i64..).zip(unrelated.iter()) {
            engine
                .add_document(
                    &mut writer,
                    &format!("22222222-2222-2222-2222-2222222222{i:02}"),
                    "codex",
                    content,
                    &[],
                    1700000000 + i,
                )
                .unwrap();
        }
        engine.commit(&mut writer).unwrap();

        let results = engine.search("smoke test from wmClient", 20).unwrap();
        assert_eq!(
            results.len(),
            1,
            "only the smoke memory should match: {results:?}"
        );
        assert_eq!(results[0].memory_id, smoke_id);
        assert!(results[0].content.contains("smoke test"));
    }

    #[test]
    fn search_project_compound_query() {
        // "antigravity antigravity-project-test" must find the project memory.
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "[antigravity:antigravity-project-test]\nQ: how does it work?\nA: details here",
                &["project_antigravity-project-test".into()],
                1700000000,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        let results = engine
            .search("antigravity antigravity-project-test", 10)
            .unwrap();
        assert!(
            !results.is_empty(),
            "project compound query must match the antigravity memory"
        );
        assert_eq!(results[0].memory_id, "11111111-1111-1111-1111-111111111111");
    }

    #[test]
    fn or_default_filters_partial_matches_via_coverage() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "alpha beta gamma delta",
                &[],
                1700000000,
            )
            .unwrap();
        engine
            .add_document(
                &mut writer,
                "22222222-2222-2222-2222-222222222222",
                "codex",
                "alpha only here",
                &[],
                1700000001,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        // OR is now the default.  A 3-term query requires 2/3 token coverage,
        // so the "alpha only here" doc (1/3) is filtered out.
        let results = engine.search("alpha beta gamma", 10).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].memory_id, "11111111-1111-1111-1111-111111111111");

        // With a 2-term query the floor is 1/2, so partial matches return.
        let results = engine.search("alpha beta", 10).unwrap();
        assert_eq!(results.len(), 2);
    }

    #[test]
    fn coverage_is_case_insensitive() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();
        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "Smoke Test for wmClient integration",
                &[],
                1700000000,
            )
            .unwrap();
        engine
            .add_document(
                &mut writer,
                "22222222-2222-2222-2222-222222222222",
                "codex",
                "test only here",
                &[],
                1700000001,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        // 3 tokens: smoke + test + wmclient (case-insensitive stemming-aware
        // whole-word matching) — only the first doc covers 2/3; the "test
        // only" doc covers 1/3 and must be dropped even though it matched
        // via OR.
        let results = engine.search("Smoke Test from wmClient", 10).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].memory_id, "11111111-1111-1111-1111-111111111111");
    }

    #[test]
    fn coverage_matches_stemmed_variants() {
        // Stemming-aware coverage: "graduate" should match "graduated",
        // "classes" should match "class", mirroring the en_stem tokenizer.
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();
        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "I graduated with a degree in Business Administration",
                &[],
                1700000000,
            )
            .unwrap();
        engine
            .add_document(
                &mut writer,
                "22222222-2222-2222-2222-222222222222",
                "codex",
                "degree only here",
                &[],
                1700000001,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        // 2-term query "graduate degree" → coverage floor 1/2.
        // Doc 1: "graduated" stems to "graduat", "degree" stems to "degre" → 2/2.
        // Doc 2: "degree" → 1/2.
        // Both pass the 1/2 floor, but doc 1 should rank higher due to
        // the coverage-ratio boost (2/2 > 1/2).
        let results = engine.search("graduate degree", 10).unwrap();
        assert_eq!(results.len(), 2);
        assert_eq!(results[0].memory_id, "11111111-1111-1111-1111-111111111111");
    }

    #[test]
    fn coverage_normalizes_possessives_and_punctuation() {
        let query = "buy sister's birthday gift";
        assert_eq!(
            query_stem_tokens(query),
            ["buy", "sister", "birthday", "gift"]
        );
        assert_eq!(
            count_token_hits("I bought a dress for my sister birthday", query),
            2
        );
    }

    #[test]
    fn search_stopword_only_query_returns_nothing() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();
        engine
            .add_document(
                &mut writer,
                "11111111-1111-1111-1111-111111111111",
                "codex",
                "some ordinary text",
                &[],
                1700000000,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        let results = engine.search("the from and or", 10).unwrap();
        assert!(results.is_empty());
    }

    #[test]
    fn wildcard_query_doesnt_match_all() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        engine
            .add_document(&mut writer, "uuid-1", "codex", "first document", &[], 1000)
            .unwrap();
        engine
            .add_document(&mut writer, "uuid-2", "codex", "second document", &[], 2000)
            .unwrap();
        engine.commit(&mut writer).unwrap();

        // Wildcard should not match all documents
        let results = engine.search("*", 10).unwrap();
        // With sanitization, "*" is treated as literal text, not wildcard
        // So it should match 0 documents (no content contains literal "*")
        assert!(
            results.is_empty(),
            "wildcard query should not match all documents after sanitization"
        );
    }

    #[test]
    fn field_syntax_query_doesnt_access_other_fields() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        // Add a doc with "secret" in galaxy field but not content
        engine
            .add_document(&mut writer, "uuid-1", "secret", "public content", &[], 1000)
            .unwrap();
        engine.commit(&mut writer).unwrap();

        // Try to use field syntax to access galaxy field
        let results = engine.search("galaxy:secret", 10).unwrap();
        // With sanitization, "galaxy:secret" is treated as literal text
        // So it should not match the galaxy field
        assert!(
            results.is_empty(),
            "field syntax injection should not access non-searchable fields"
        );
    }

    #[test]
    fn boolean_operator_doesnt_bypass_search() {
        let (_tmp, engine) = open_engine();
        let mut writer = engine.writer().unwrap();

        engine
            .add_document(
                &mut writer,
                "uuid-1",
                "codex",
                "important secret data",
                &[],
                1000,
            )
            .unwrap();
        engine.commit(&mut writer).unwrap();

        // Uppercase operator words are quoted (and stripped as stopwords), so
        // they cannot be used to exclude or require terms: "AND secret"
        // reduces to a plain search for "secret" and must still match.
        let results = engine.search("AND secret", 10).unwrap();
        assert_eq!(results.len(), 1);

        // A query made only of operators/stopwords has no terms and matches
        // nothing — it cannot match the whole corpus.
        let results = engine.search("AND OR NOT", 10).unwrap();
        assert!(
            results.is_empty(),
            "operator-only query must not bypass search"
        );
    }
}