eidetic-engine 0.18.0

//! Small, total parser for agent-facing search query strings.
//!
//! The parser intentionally accepts every UTF-8 string and emits a canonical
//! printable form. Retrieval engines may still interpret the resulting terms
//! differently, but this boundary gives property and fuzz tests a narrow target
//! for query normalization.
//! Bare terms accept `\-` and `\\` at the start to preserve a literal leading
//! exclusion marker or backslash. These escapes apply only at the start of a term.

use std::fmt;
use std::iter::Peekable;
use std::str::Chars;

use super::scoring::AnchorMatchContext;
use crate::models::{MemoryAnchorSource, extract_precision_memory_anchors};

/// Parsed search query with deterministic clause ordering.
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct ParsedSearchQuery {
    clauses: Vec<SearchQueryClause>,
}

impl ParsedSearchQuery {
    #[must_use]
    pub fn clauses(&self) -> &[SearchQueryClause] {
        &self.clauses
    }

    #[must_use]
    pub fn is_empty(&self) -> bool {
        self.clauses.is_empty()
    }
}

impl fmt::Display for ParsedSearchQuery {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        for (index, clause) in self.clauses.iter().enumerate() {
            if index > 0 {
                formatter.write_str(" ")?;
            }
            fmt::Display::fmt(clause, formatter)?;
        }
        Ok(())
    }
}

/// One canonical search query clause.
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum SearchQueryClause {
    Term(String),
    Phrase(String),
    ExcludedTerm(String),
    ExcludedPhrase(String),
}

impl fmt::Display for SearchQueryClause {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Term(term) => write_printable_unquoted(term, true, formatter),
            Self::Phrase(phrase) => write_quoted(phrase, formatter),
            Self::ExcludedTerm(term) => {
                formatter.write_str("-")?;
                write_printable_unquoted(term, false, formatter)
            }
            Self::ExcludedPhrase(phrase) => {
                formatter.write_str("-")?;
                write_quoted(phrase, formatter)
            }
        }
    }
}

/// Parse a search query string into a canonical, printable representation.
///
/// This function is total over UTF-8 input: malformed or incomplete syntax is
/// normalized into ordinary terms/phrases rather than returned as an error.
#[must_use]
pub fn parse_search_query(input: &str) -> ParsedSearchQuery {
    let mut chars = input.chars().peekable();
    let mut clauses = Vec::new();

    while chars.peek().is_some() {
        skip_separators(&mut chars);
        if chars.peek().is_none() {
            break;
        }
        if let Some(clause) = parse_clause(&mut chars) {
            clauses.push(clause);
        }
    }

    ParsedSearchQuery { clauses }
}

fn parse_clause(chars: &mut Peekable<Chars<'_>>) -> Option<SearchQueryClause> {
    let excluded = if matches!(chars.peek(), Some('-')) {
        chars.next();
        true
    } else {
        false
    };

    let quoted = matches!(chars.peek(), Some('"'));
    let value = if quoted {
        chars.next();
        if !remaining_contains_closing_quote(chars) {
            let value = parse_unclosed_quoted_bare(chars);
            if excluded && value.is_empty() {
                return Some(SearchQueryClause::Term("-".to_string()));
            }
            if value.is_empty() {
                return None;
            }
            return if excluded {
                Some(SearchQueryClause::ExcludedTerm(value))
            } else {
                Some(SearchQueryClause::Term(value))
            };
        }
        parse_quoted(chars)
    } else {
        parse_bare(chars)
    };

    if excluded && !quoted && value.is_empty() {
        return Some(SearchQueryClause::Term("-".to_string()));
    }

    if value.is_empty() {
        return None;
    }

    match (excluded, quoted) {
        (false, false) => Some(SearchQueryClause::Term(value)),
        (false, true) => Some(SearchQueryClause::Phrase(value)),
        (true, false) => Some(SearchQueryClause::ExcludedTerm(value)),
        (true, true) => Some(SearchQueryClause::ExcludedPhrase(value)),
    }
}

fn skip_separators(chars: &mut Peekable<Chars<'_>>) {
    while matches!(chars.peek(), Some(value) if is_query_separator(*value)) {
        chars.next();
    }
}

fn parse_bare(chars: &mut Peekable<Chars<'_>>) -> String {
    let mut value = String::new();
    // A leading escape keeps a literal exclusion marker (or the escape itself)
    // from acquiring syntax when a parsed term is printed and parsed again.
    if matches!(chars.peek(), Some('\\')) {
        let mut lookahead = chars.clone();
        lookahead.next();
        if matches!(lookahead.peek(), Some('-' | '\\')) {
            chars.next();
            if let Some(escaped) = chars.next() {
                value.push(escaped);
            }
        }
    }
    while let Some(next) = chars.peek().copied() {
        if is_query_separator(next) || next == '"' {
            break;
        }
        if next == '\\' {
            let mut lookahead = chars.clone();
            lookahead.next();
            if matches!(lookahead.peek(), Some('"')) {
                chars.next();
                chars.next();
                break;
            }
        }
        value.push(next);
        chars.next();
    }
    value
}

fn parse_unclosed_quoted_bare(chars: &mut Peekable<Chars<'_>>) -> String {
    let mut value = String::new();
    while let Some(next) = chars.peek().copied() {
        if is_query_separator(next) {
            break;
        }
        if next == '"' {
            break;
        }
        if next == '\\' {
            chars.next();
            if matches!(chars.peek(), Some('"')) {
                chars.next();
                break;
            }
            value.push(next);
            continue;
        }
        value.push(next);
        chars.next();
    }
    value
}

fn remaining_contains_closing_quote(chars: &Peekable<Chars<'_>>) -> bool {
    let mut escaped = false;
    for next in chars.clone() {
        if escaped {
            escaped = false;
            continue;
        }
        if next == '\\' {
            escaped = true;
            continue;
        }
        if next == '"' {
            return true;
        }
    }
    false
}

fn parse_quoted(chars: &mut Peekable<Chars<'_>>) -> String {
    let mut value = String::new();
    let mut last_was_normalized_space = false;
    while let Some(next) = chars.next() {
        match next {
            '"' => break,
            '\\' => match chars.next() {
                Some(escaped) => {
                    push_quoted_printable(&mut value, escaped, &mut last_was_normalized_space)
                }
                None => value.push('\\'),
            },
            other => push_quoted_printable(&mut value, other, &mut last_was_normalized_space),
        }
    }
    value
}

fn is_query_separator(character: char) -> bool {
    character.is_whitespace() || character.is_control()
}

fn push_quoted_printable(
    value: &mut String,
    character: char,
    last_was_normalized_space: &mut bool,
) {
    if character.is_control() {
        if !*last_was_normalized_space {
            value.push(' ');
            *last_was_normalized_space = true;
        }
    } else {
        value.push(character);
        *last_was_normalized_space = false;
    }
}

fn write_quoted(value: &str, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
    formatter.write_str("\"")?;
    let mut last_was_normalized_space = false;
    for character in value.chars() {
        match character {
            '"' => formatter.write_str("\\\"")?,
            '\\' => formatter.write_str("\\\\")?,
            other if other.is_control() => {
                if !last_was_normalized_space {
                    formatter.write_str(" ")?;
                    last_was_normalized_space = true;
                }
            }
            other => write!(formatter, "{other}")?,
        }
        if !character.is_control() {
            last_was_normalized_space = false;
        }
    }
    formatter.write_str("\"")
}

fn write_printable_unquoted(
    value: &str,
    escape_exclusion: bool,
    formatter: &mut fmt::Formatter<'_>,
) -> fmt::Result {
    if (escape_exclusion && value.starts_with('-') && value != "-")
        || value.starts_with("\\-")
        || value.starts_with("\\\\")
    {
        formatter.write_str("\\")?;
    }
    let mut last_was_normalized_space = false;
    for character in value.chars() {
        if character.is_control() {
            if !last_was_normalized_space {
                formatter.write_str(" ")?;
                last_was_normalized_space = true;
            }
        } else {
            write!(formatter, "{character}")?;
            last_was_normalized_space = false;
        }
    }
    Ok(())
}

/// Build the redaction-safe anchor-match context for a search query
/// (bd-1n0np.3.3).
///
/// Runs the same precision anchor extraction used at ingest over the query
/// string and projects each anchor to a `(kind, hash)` pair. The result feeds
/// [`anchor_match_score`](super::scoring::anchor_match_score): a query that
/// names exact code surfaces (`src/db/mod.rs`, schema `ee.response.v2`,
/// `EE_PACK_TRACE`, `cargo fmt --check`, …) can additively boost already-
/// retrieved candidates carrying the same hashed anchors, while Frankensearch
/// keeps ownership of candidate selection and base ranking. Only hashes cross
/// this boundary — raw query surfaces are never compared or stored. Prose with
/// no exact code surface yields a cold-start context (no boost).
#[must_use]
pub fn query_anchor_match_context(query: &str) -> AnchorMatchContext {
    let anchors = extract_precision_memory_anchors(
        "mem_searchquery00000000000000000",
        query,
        MemoryAnchorSource::Explicit,
        Some("search.query"),
    );
    AnchorMatchContext::new(anchors.into_iter().map(|anchor| {
        (
            anchor.anchor_kind.as_str().to_owned(),
            anchor.anchor_value_hash,
        )
    }))
}

/// The query text handed to the lexical engine for an agent's search.
///
/// The engine's parser reads a leading `-` as "exclude this term", so a query
/// such as `cargo fmt --check before release` excluded every memory that
/// mentions `check`, including the one it described (bd-bka39). Agents quote
/// command lines far more often than they mean exclusion, so a token shaped
/// like a command-line option is searched as the word it names: a long option
/// (`--check`, `--no-verify`) or a short flag cluster of at most three letters
/// or digits (`-rf`, `-p`, `-j8`, `-1`). A longer single-dash word
/// (`-legacy`) keeps its exclusion meaning, and text inside double quotes is
/// left alone.
///
/// The parser also reads `word:` as "search field `word`", and a query naming
/// a field the index does not have matched nothing at all: `std::fs::read`,
/// `File::open` and `src/core/search.rs:120` returned no results. A colon
/// outside quotes is a field prefix only after a real field name (`id`,
/// `content`, `title`); anywhere else it separates words. Everything else
/// passes through unchanged.
///
/// Walk option values through the same quote/escape state as ordinary text.
/// Consuming a whole option at once would miss an opening quote, preserve an
/// unknown field inside its value, or swallow a closing query group. Only an
/// original single-colon field prefix has field semantics: removing dashes or
/// splitting a Rust path must not manufacture a new `title:`/`content:` filter.
#[must_use]
pub fn lexical_engine_query(query: &str) -> std::borrow::Cow<'_, str> {
    if !query.contains(['-', ':']) {
        return std::borrow::Cow::Borrowed(query);
    }
    let mut out = String::with_capacity(query.len());
    let mut in_quote = false;
    let mut escaped = false;
    let mut at_token_start = true;
    let mut token_start = 0;
    let mut field_allowed = true;
    let mut rest = query;
    while let Some(ch) = rest.chars().next() {
        if escaped {
            // Preserve the downstream parser's escape verbatim. In particular,
            // an escaped quote is not the start/end of a phrase.
            out.push(ch);
            escaped = false;
            rest = &rest[ch.len_utf8()..];
            continue;
        }
        if ch == '\\' {
            out.push(ch);
            escaped = true;
            at_token_start = false;
            field_allowed = false;
            rest = &rest[1..];
            continue;
        }
        if ch == '"' {
            in_quote = !in_quote;
            out.push(ch);
            at_token_start = false;
            field_allowed = false;
            rest = &rest[1..];
            continue;
        }
        if in_quote {
            out.push(ch);
            rest = &rest[ch.len_utf8()..];
            continue;
        }
        if ch == ':' {
            let field = out[token_start..].trim_start_matches(['+', '-']);
            let real_field = field_allowed
                && matches!(field, "id" | "content" | "title")
                && !rest.starts_with("::");
            out.push(if real_field { ':' } else { ' ' });
            // A field's value, an option value, or the tail of a technical
            // path cannot become another field merely because we split it.
            field_allowed = false;
            at_token_start = true;
            token_start = out.len();
            rest = &rest[1..];
            continue;
        }
        if ch == '-' && at_token_start {
            let dashes = rest.len() - rest.trim_start_matches('-').len();
            let token_end = rest
                .find(|ch: char| ch.is_whitespace() || matches!(ch, '(' | ')' | '"'))
                .unwrap_or(rest.len());
            let word = &rest[dashes..token_end];
            let short_flag = dashes == 1
                && (1..=3).contains(&word.len())
                && word.bytes().all(|byte| byte.is_ascii_alphanumeric());
            let long_option = dashes >= 2
                && word
                    .chars()
                    .next()
                    .is_some_and(|first| first.is_alphanumeric());
            if short_flag || long_option {
                // Strip only the prefix. Colons, quoted option values and
                // group delimiters still pass through the common state machine.
                field_allowed = false;
            } else if dashes >= 2
                && word.is_empty()
                && rest[dashes..]
                    .chars()
                    .next()
                    .is_none_or(|next| next.is_whitespace() || next == ')')
            {
                // A bare `--` ends options in a command line; it names nothing.
                field_allowed = false;
            } else {
                out.push_str(&rest[..dashes]);
            }
            rest = &rest[dashes..];
            at_token_start = false;
            continue;
        }
        out.push(ch);
        if ch.is_whitespace() || matches!(ch, '(' | ')') {
            at_token_start = true;
            field_allowed = true;
            token_start = out.len();
        } else if ch == '+' && at_token_start {
            // Required-clause syntax does not turn the following option into
            // an exclusion, and must not hide an otherwise valid field name.
            token_start = out.len();
        } else {
            at_token_start = false;
        }
        rest = &rest[ch.len_utf8()..];
    }
    if out == query {
        std::borrow::Cow::Borrowed(query)
    } else {
        std::borrow::Cow::Owned(out)
    }
}

#[cfg(test)]
mod tests {
    use super::{
        SearchQueryClause, lexical_engine_query, parse_search_query, query_anchor_match_context,
    };

    #[test]
    fn search_query_parser_normalizes_terms_phrases_and_exclusions() {
        let query = parse_search_query(r#"  release  "cargo fmt" -"bad idea"  --flag "#);

        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Term("release".to_string()),
                SearchQueryClause::Phrase("cargo fmt".to_string()),
                SearchQueryClause::ExcludedPhrase("bad idea".to_string()),
                SearchQueryClause::ExcludedTerm("-flag".to_string()),
            ]
        );
        assert_eq!(
            query.to_string(),
            r#"release "cargo fmt" -"bad idea" --flag"#
        );
    }

    #[test]
    fn search_query_parser_roundtrips_escaped_phrases() {
        let query = parse_search_query(r#""quoted \"value\" and \\ slash""#);
        let printed = query.to_string();

        assert_eq!(parse_search_query(&printed), query);
    }

    #[test]
    fn search_query_parser_preserves_dangling_exclusion_marker() {
        let query = parse_search_query("alpha - beta -");

        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Term("alpha".to_string()),
                SearchQueryClause::Term("-".to_string()),
                SearchQueryClause::Term("beta".to_string()),
                SearchQueryClause::Term("-".to_string()),
            ]
        );
        assert_eq!(query.to_string(), "alpha - beta -");
        assert_eq!(parse_search_query(&query.to_string()), query);
    }

    #[test]
    fn search_query_parser_roundtrips_literal_exclusion_after_unclosed_quotes() {
        let input = String::from_utf8(vec![34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 45, 33])
            .expect("regression input is ASCII");
        let query = parse_search_query(&input);

        assert_eq!(query.clauses(), &[SearchQueryClause::Term("-!".into())]);
        assert_eq!(query.to_string(), r"\-!");
        assert_eq!(parse_search_query(&query.to_string()), query);

        let query = parse_search_query("\"-! alpha -beta");
        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Term("-!".into()),
                SearchQueryClause::Term("alpha".into()),
                SearchQueryClause::ExcludedTerm("beta".into()),
            ]
        );
        assert_eq!(query.to_string(), r"\-! alpha -beta");
        assert_eq!(parse_search_query(&query.to_string()), query);
    }

    #[test]
    fn search_query_parser_roundtrips_leading_bare_escapes() {
        for value in [
            "-",
            "-!",
            "--flag",
            r"\",
            r"\-!",
            r"\\server",
            r"path:\name",
        ] {
            for clause in [
                SearchQueryClause::Term(value.into()),
                SearchQueryClause::ExcludedTerm(value.into()),
            ] {
                let printed = clause.to_string();
                let query = parse_search_query(&printed);
                assert_eq!(query.clauses(), &[clause], "printed: {printed:?}");
                assert_eq!(query.to_string(), printed);
            }
        }

        let query = parse_search_query(r"\-! -\-! \\-! -\\-!");
        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Term("-!".into()),
                SearchQueryClause::ExcludedTerm("-!".into()),
                SearchQueryClause::Term(r"\-!".into()),
                SearchQueryClause::ExcludedTerm(r"\-!".into()),
            ]
        );
        assert_eq!(parse_search_query(&query.to_string()), query);
    }

    #[test]
    fn search_query_parser_still_drops_empty_quoted_clauses() {
        let query = parse_search_query(r#"alpha "" -"" beta"#);

        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Term("alpha".to_string()),
                SearchQueryClause::Term("beta".to_string()),
            ]
        );
        assert_eq!(query.to_string(), "alpha beta");
    }

    #[test]
    fn search_query_parser_splits_adjacent_bare_and_quoted_clauses() {
        let query = parse_search_query(r#"alpha"quoted value"beta"#);

        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Term("alpha".to_string()),
                SearchQueryClause::Phrase("quoted value".to_string()),
                SearchQueryClause::Term("beta".to_string()),
            ]
        );
        assert_eq!(query.to_string(), r#"alpha "quoted value" beta"#);
        assert_eq!(parse_search_query(&query.to_string()), query);
    }

    #[test]
    fn search_query_parser_drops_dangling_quote_without_raw_quote_in_term() {
        let query = parse_search_query(r#"alpha" beta"#);

        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Term("alpha".to_string()),
                SearchQueryClause::Term("beta".to_string()),
            ]
        );
        assert_eq!(query.to_string(), "alpha beta");
    }

    #[test]
    fn search_query_parser_treats_unclosed_phrase_as_terms() {
        let query = parse_search_query(r#""alpha beta -gamma"#);

        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Term("alpha".to_string()),
                SearchQueryClause::Term("beta".to_string()),
                SearchQueryClause::ExcludedTerm("gamma".to_string()),
            ]
        );
        assert_eq!(query.to_string(), "alpha beta -gamma");
        assert_eq!(parse_search_query(&query.to_string()), query);
    }

    #[test]
    fn search_query_parser_does_not_emit_escape_marker_for_unclosed_phrase() {
        let query = parse_search_query(r#"alpha "beta \" gamma"#);

        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Term("alpha".to_string()),
                SearchQueryClause::Term("beta".to_string()),
                SearchQueryClause::Term("gamma".to_string()),
            ]
        );
        assert_eq!(query.to_string(), "alpha beta gamma");
        assert_eq!(parse_search_query(&query.to_string()), query);
    }

    #[test]
    fn search_query_parser_keeps_exclusion_for_dangling_quoted_term() {
        let query = parse_search_query(r#"-"alpha beta"#);

        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::ExcludedTerm("alpha".to_string()),
                SearchQueryClause::Term("beta".to_string()),
            ]
        );
        assert_eq!(query.to_string(), "-alpha beta");
        assert_eq!(parse_search_query(&query.to_string()), query);
    }

    #[test]
    fn search_query_parser_treats_unbalanced_groups_and_brackets_as_literals() {
        let query = parse_search_query(r#"title:[release TO beta) (dangling [open] -scope:(bad]"#);

        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Term("title:[release".to_string()),
                SearchQueryClause::Term("TO".to_string()),
                SearchQueryClause::Term("beta)".to_string()),
                SearchQueryClause::Term("(dangling".to_string()),
                SearchQueryClause::Term("[open]".to_string()),
                SearchQueryClause::ExcludedTerm("scope:(bad]".to_string()),
            ]
        );
        assert_eq!(
            query.to_string(),
            r#"title:[release TO beta) (dangling [open] -scope:(bad]"#
        );
        assert_eq!(parse_search_query(&query.to_string()), query);
    }

    #[test]
    fn search_query_parser_unescapes_bracket_like_characters_only_inside_phrases() {
        let query = parse_search_query(r#""escaped \[brackets\] and \(group\)" path:\[literal\]"#);

        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Phrase("escaped [brackets] and (group)".to_string()),
                SearchQueryClause::Term(r#"path:\[literal\]"#.to_string()),
            ]
        );
        assert_eq!(
            query.to_string(),
            r#""escaped [brackets] and (group)" path:\[literal\]"#
        );
        assert_eq!(parse_search_query(&query.to_string()), query);
    }

    #[test]
    fn search_query_parser_preserves_unicode_format_controls_and_replacement_chars() {
        let query =
            parse_search_query("field\u{200d}:value bad\u{fffd}field -\"zero\u{200d} width\"");

        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Term("field\u{200d}:value".to_string()),
                SearchQueryClause::Term("bad\u{fffd}field".to_string()),
                SearchQueryClause::ExcludedPhrase("zero\u{200d} width".to_string()),
            ]
        );
        assert_eq!(
            query.to_string(),
            "field\u{200d}:value bad\u{fffd}field -\"zero\u{200d} width\""
        );
        assert_eq!(parse_search_query(&query.to_string()), query);
    }

    #[test]
    fn search_query_parser_normalizes_ascii_controls_to_printable_form() {
        let query =
            parse_search_query("alpha\u{0000}beta \"line\u{0007}\nfeed\" -gamma\u{001f}delta");

        assert_eq!(
            query.clauses(),
            &[
                SearchQueryClause::Term("alpha".to_string()),
                SearchQueryClause::Term("beta".to_string()),
                SearchQueryClause::Phrase("line feed".to_string()),
                SearchQueryClause::ExcludedTerm("gamma".to_string()),
                SearchQueryClause::Term("delta".to_string()),
            ]
        );
        assert_eq!(query.to_string(), r#"alpha beta "line feed" -gamma delta"#);
        assert!(
            !query.to_string().chars().any(char::is_control),
            "canonical query form must not emit raw control characters"
        );
        assert_eq!(parse_search_query(&query.to_string()), query);
    }

    #[test]
    fn search_query_clause_display_sanitizes_direct_control_char_values() {
        for clause in [
            SearchQueryClause::Term("alpha\u{0000}\nbeta".to_string()),
            SearchQueryClause::Phrase("line\u{0007}\nfeed".to_string()),
            SearchQueryClause::ExcludedTerm("gamma\u{001f}delta".to_string()),
            SearchQueryClause::ExcludedPhrase("tab\tfeed".to_string()),
        ] {
            assert!(
                !clause.to_string().chars().any(char::is_control),
                "direct clause Display must not emit raw controls: {clause:?}"
            );
        }

        assert_eq!(
            SearchQueryClause::Phrase("line\u{0007}\nfeed".to_string()).to_string(),
            r#""line feed""#
        );
        assert_eq!(
            SearchQueryClause::ExcludedTerm("gamma\u{001f}delta".to_string()).to_string(),
            "-gamma delta"
        );
    }

    #[test]
    fn query_anchor_match_context_extracts_hashed_surfaces() {
        let context = query_anchor_match_context("Edit `src/db/mod.rs` and honor `EE_PACK_TRACE`");
        assert!(!context.is_cold_start());
        // The query's path surface is carried as its canonical anchor hash,
        // matching what extraction stored for the same path.
        let path_hash = crate::models::memory_anchor_value_hash(
            crate::models::MemoryAnchorKind::Path,
            "src/db/mod.rs",
        );
        assert!(context.anchors.contains(&("path".to_owned(), path_hash)));
        // No raw surface string leaks into the context.
        assert!(
            context
                .anchors
                .iter()
                .all(|(_, hash)| hash.starts_with("blake3:") && !hash.contains("src/db/mod.rs"))
        );
        // Prose with no exact code surface yields a cold-start context (no boost).
        assert!(query_anchor_match_context("just ordinary prose words").is_cold_start());
    }

    #[test]
    fn lexical_engine_query_searches_command_line_options_as_words() {
        for (query, expected) in [
            (
                "Always run cargo fmt --check before release",
                "Always run cargo fmt check before release",
            ),
            ("rm -rf target", "rm rf target"),
            (
                "cargo test -p ee -j8 -- --nocapture",
                "cargo test p ee j8  nocapture",
            ),
            ("git commit --no-verify", "git commit no-verify"),
            ("retry with -1 offset", "retry with 1 offset"),
            ("release -legacy", "release -legacy"),
            ("plain words only", "plain words only"),
            (
                "\"cargo fmt --check\" release",
                "\"cargo fmt --check\" release",
            ),
            ("(--check OR fmt)", "(check OR fmt)"),
            ("well-known x-ray", "well-known x-ray"),
            ("-", "-"),
            ("std::fs::read_to_string", "std  fs  read_to_string"),
            ("File::open failed", "File  open failed"),
            ("src/core/search.rs:120", "src/core/search.rs 120"),
            ("title:release notes", "title:release notes"),
            ("(content:fmt)", "(content:fmt)"),
            ("\"a::b\" c", "\"a::b\" c"),
            ("Widget: Serialize", "Widget  Serialize"),
        ] {
            assert_eq!(lexical_engine_query(query), expected, "{query}");
        }
        assert!(matches!(
            lexical_engine_query("no dashes here"),
            std::borrow::Cow::Borrowed(_)
        ));
    }

    #[test]
    fn lexical_options_do_not_bypass_quoted_values_or_excluded_phrases() {
        for (query, expected) in [
            (
                r#"--message="std::fs --check" File::open"#,
                r#"message="std::fs --check" File  open"#,
            ),
            (
                r#"release -"legacy --no-verify" --check"#,
                r#"release -"legacy --no-verify" check"#,
            ),
            (
                r#"--message="quoted \"value\" --flag" --check"#,
                r#"message="quoted \"value\" --flag" check"#,
            ),
            (
                r#"--message="unfinished --check std::fs"#,
                r#"message="unfinished --check std::fs"#,
            ),
        ] {
            assert_eq!(lexical_engine_query(query), expected, "{query}");
        }
    }

    #[test]
    fn lexical_escapes_never_change_the_phrase_state() {
        for (query, expected) in [
            (r#"\" --check File::open"#, r#"\" check File  open"#),
            (
                r#""before \" --check std::fs" --verify"#,
                r#""before \" --check std::fs" verify"#,
            ),
            (r"\-legacy --check", r"\-legacy check"),
            (r"title\:release --check", r"title\:release check"),
            (r"C:\repo\cache.rs:12 --check", r"C \repo\cache.rs 12 check"),
        ] {
            assert_eq!(lexical_engine_query(query), expected, "{query}");
        }
        for count in 0..=8 {
            let slashes = "\\".repeat(count);
            let query = format!("{slashes}\"--check std::fs");
            let expected = if count % 2 == 0 {
                query.clone()
            } else {
                // An escaped quote remains part of the current bare token;
                // its dashes are not an independent option prefix.
                format!("{slashes}\"--check std  fs")
            };
            assert_eq!(lexical_engine_query(&query), expected, "{count} slashes");
        }
    }

    #[test]
    fn lexical_field_and_group_boundaries_keep_options_searchable() {
        for (query, expected) in [
            ("content:--check", "content:check"),
            ("title:-j8", "title:j8"),
            ("content:(-p OR -j8)", "content:(p OR j8)"),
            ("(+--check OR (-rf))", "(+check OR (rf))"),
            ("(-p) OR (--check)", "(p) OR (check)"),
            ("-- --check", " check"),
            ("(--)", "()"),
        ] {
            assert_eq!(lexical_engine_query(query), expected, "{query}");
        }
        for query in [
            "release -legacy",
            r#"release -"legacy release""#,
            "+title:release -content:legacy",
            r#"content:"cargo fmt --check""#,
            "(title:release OR content:pack)",
        ] {
            assert_eq!(lexical_engine_query(query), query);
        }
    }

    #[test]
    fn lexical_technical_colons_cannot_invent_reserved_field_filters() {
        for (query, expected) in [
            ("id::from_str", "id  from_str"),
            ("title::render", "title  render"),
            ("std::content:render", "std  content render"),
            ("--title:release", "title release"),
            ("--filter=unknown:release", "filter=unknown release"),
            ("--target=std::fs", "target=std  fs"),
            ("--check title:release", "check title:release"),
            ("title:release content:pack", "title:release content:pack"),
        ] {
            assert_eq!(lexical_engine_query(query), expected, "{query}");
        }
    }

    #[test]
    fn lexical_normalization_preserves_unicode_source_and_is_idempotent() {
        let pieces = [
            "資料",
            "🦀",
            "--check",
            "-legacy",
            "id::read",
            "title:release",
            r#"\""#,
            "\"",
            "(",
            ")",
            " ",
            ":",
            "\\",
        ];
        for left in pieces {
            for middle in pieces {
                for right in pieces {
                    let query = format!("{left}{middle}{right}");
                    let source = query.clone();
                    let normalized = lexical_engine_query(&query);
                    assert_eq!(query, source, "source query was rewritten");
                    assert!(normalized.len() <= query.len());
                    assert_eq!(
                        lexical_engine_query(&normalized),
                        normalized,
                        "normalization drifted for {query:?}"
                    );
                }
            }
        }
    }

    #[cfg(feature = "lexical-bm25")]
    #[test]
    fn lexical_command_queries_retrieve_the_expected_backend_documents() -> Result<(), String> {
        use crate::search::{IndexableDocument, LexicalRead, LexicalWrite, TantivyIndex};
        use std::collections::BTreeSet;

        crate::core::run_cli_future(async {
            let cx = asupersync::Cx::current().ok_or("test runtime context missing")?;
            let index = TantivyIndex::in_memory().map_err(|error| error.to_string())?;
            let mut documents = vec![
                IndexableDocument::new("evidence-command", "Run cargo fmt --check before release."),
                IndexableDocument::new(
                    "evidence-legacy",
                    "Run cargo fmt --check before release. legacy --no-verify",
                ),
                IndexableDocument::new("evidence-flags", "Run cargo test -p ee -j8."),
                IndexableDocument::new(
                    "evidence-symbol",
                    "id::from_str resolves source identities. Use std::fs::read for source bytes.",
                ),
                IndexableDocument::new("title-only", "An unrelated packaging observation.")
                    .with_title("check"),
            ];
            for document in &mut documents {
                document
                    .metadata
                    .insert("source_marker".to_owned(), document.id.clone());
            }
            index
                .index_documents(&cx, &documents)
                .await
                .map_err(|error| error.to_string())?;
            index.commit(&cx).await.map_err(|error| error.to_string())?;
            for (query, expected) in [
                (
                    "content:--check",
                    vec!["evidence-command", "evidence-legacy"],
                ),
                ("content:--check -legacy", vec!["evidence-command"]),
                (
                    r#"content:--check -"legacy --no-verify""#,
                    vec!["evidence-command"],
                ),
                ("content:(-p OR -j8)", vec!["evidence-flags"]),
                ("id::from_str", vec!["evidence-symbol"]),
                ("--target=std::fs", vec!["evidence-symbol"]),
            ] {
                let normalized = lexical_engine_query(query);
                let hits = index
                    .search(&cx, &normalized, 10)
                    .await
                    .map_err(|error| error.to_string())?;
                let actual: BTreeSet<_> = hits.iter().map(|hit| hit.doc_id.to_string()).collect();
                let expected: BTreeSet<_> = expected.into_iter().map(str::to_owned).collect();
                assert_eq!(actual, expected, "query: {query}; normalized: {normalized}");
                for hit in &hits {
                    let wire = serde_json::to_value(hit).map_err(|error| error.to_string())?;
                    assert_eq!(
                        wire["metadata"]["source_marker"].as_str(),
                        Some(hit.doc_id.as_str())
                    );
                }
            }
            Ok(())
        })
        .map_err(|error| error.to_string())?
    }
}