luci/query/

phrase.rs

//! MatchPhraseQuery: terms must appear in order at consecutive positions.
//!
//! Uses position posting lists directly for both conjunction and position
//! verification, eliminating the overhead of maintaining separate posting
//! lists. The lead reader (fewest docs) drives iteration; other readers
//! advance() to each candidate doc, skipping intermediate positions.
//!
//! See [[query-dsl#Full-Text Queries]] and [[architecture-query-execution#Step 9]].

use crate::core::{DocId, FieldId, NO_MORE_DOCS, Result, ScoreMode, Scorer, TwoPhaseIterator};

use crate::query::term::TermQuery;
use crate::query::{BoundQuery, Query, ScorerSupplier};
use crate::search::bm25::{bm25_idf, bm25_score};
use crate::search::searcher::Searcher;
use crate::segment::reader::SegmentReader;

pub struct MatchPhraseQuery {
    pub field: String,
    pub query_text: String,
    pub analyzer: Option<String>,
}

impl Query for MatchPhraseQuery {
    fn bind(&self, searcher: &Searcher, score_mode: ScoreMode) -> Result<Box<dyn BoundQuery>> {
        let analyzer_name = searcher.resolve_search_analyzer(&self.field, self.analyzer.as_deref());
        let analyzer = searcher.analyzers().get(analyzer_name);
        let tokens = analyzer.analyze(&self.query_text);

        if tokens.is_empty() {
            return Ok(Box::new(BoundEmptyQuery));
        }

        if tokens.len() == 1 {
            let tq = TermQuery {
                field: self.field.clone(),
                value: tokens[0].text.clone(),
            };
            return tq.bind(searcher, score_mode);
        }

        let terms: Vec<String> = tokens.iter().map(|t| t.text.clone()).collect();

        Ok(Box::new(BoundPhraseQuery {
            field: self.field.clone(),
            terms,
            total_docs: searcher.total_docs(),
            avg_field_length: searcher.avg_field_length(&self.field),
        }))
    }
}

struct BoundPhraseQuery {
    field: String,
    terms: Vec<String>,
    total_docs: u32,
    avg_field_length: f32,
}

impl BoundQuery for BoundPhraseQuery {
    fn scorer_supplier(&self, reader: &SegmentReader) -> Result<Option<Box<dyn ScorerSupplier>>> {
        let field_id = match reader
            .header()
            .fields
            .iter()
            .find(|f| f.field_name == self.field)
            .map(|f| f.field_id)
        {
            Some(id) => id,
            None => return Ok(None),
        };

        // All terms must exist for the phrase to match
        let mut term_doc_freqs = Vec::new();
        for term in &self.terms {
            let df = reader.doc_freq(field_id, term);
            if df == 0 {
                return Ok(None);
            }
            term_doc_freqs.push(df);
        }

        let cost = *term_doc_freqs.iter().min().unwrap() as u64;

        Ok(Some(Box::new(PhraseScorerSupplier {
            field_id,
            terms: self.terms.clone(),
            term_doc_freqs,
            total_docs: self.total_docs,
            avg_field_length: self.avg_field_length,
            cost,
            segment_data: reader as *const SegmentReader,
        })))
    }
}

struct PhraseScorerSupplier {
    field_id: FieldId,
    terms: Vec<String>,
    term_doc_freqs: Vec<u32>,
    total_docs: u32,
    avg_field_length: f32,
    cost: u64,
    segment_data: *const SegmentReader,
}

unsafe impl Send for PhraseScorerSupplier {}

impl ScorerSupplier for PhraseScorerSupplier {
    fn cost(&self) -> u64 {
        self.cost
    }

    fn scorer(self: Box<Self>) -> Result<Box<dyn Scorer>> {
        let reader = unsafe { &*self.segment_data };

        // Open position posting lists — used for BOTH conjunction and
        // position verification. This eliminates the separate simple posting
        // lists that the old approach maintained in parallel.
        let mut position_readers: Vec<crate::inverted::postings::PositionPostingListReader> =
            Vec::new();
        for term in &self.terms {
            match reader.postings_with_positions(self.field_id, term) {
                Some(r) => position_readers.push(r),
                None => return Ok(Box::new(SimpleIterScorer::empty())),
            }
        }

        // Sort by doc frequency (cheapest = fewest docs = lead iterator)
        // Use index mapping to track original term order for position offsets
        let mut term_order: Vec<usize> = (0..self.terms.len()).collect();
        term_order.sort_by_key(|&i| self.term_doc_freqs[i]);

        let sorted_readers: Vec<_> = term_order
            .iter()
            .map(|&i| {
                std::mem::replace(
                    &mut position_readers[i],
                    crate::inverted::postings::PositionPostingListReader::new(&[]),
                )
            })
            .collect();
        let sorted_offsets: Vec<u32> = term_order.iter().map(|&i| i as u32).collect();

        // Compute phrase IDF as sum of individual IDFs
        let idf: f32 = self
            .term_doc_freqs
            .iter()
            .map(|&df| bm25_idf(self.total_docs, df))
            .sum();

        // Pre-read first entry from lead reader
        let reader_state: Vec<(u32, Vec<u32>)> = sorted_readers
            .iter()
            .map(|_| (u32::MAX, Vec::new()))
            .collect();

        let mut scorer = PhraseScorer {
            readers: sorted_readers,
            term_offsets: sorted_offsets,
            reader_state,
            current: NO_MORE_DOCS,
            idf,
            avg_field_length: self.avg_field_length,
            norms: reader.norms(self.field_id),
            constant_score: reader
                .norms(self.field_id)
                .and_then(|n| n.uniform_norm())
                .map(|dl| bm25_score(idf, 1.0, dl, self.avg_field_length)),
            ptrs_buf: Vec::new(),
            phrase_freq: 0,
        };

        // Position on the first phrase match
        scorer.advance_to_next_phrase();

        Ok(Box::new(scorer))
    }
}

/// Phrase scorer using position posting lists directly for both conjunction
/// and position verification. Eliminates the separate simple posting lists
/// that the old two-phase approach maintained in parallel.
///
/// The lead reader (fewest docs) drives iteration. For each lead doc, all
/// other readers advance to that doc. If all terms are present, positions
/// are checked for consecutive sequence.
struct PhraseScorer<'a> {
    /// Position readers sorted by doc frequency (lead = index 0).
    readers: Vec<crate::inverted::postings::PositionPostingListReader<'a>>,
    /// Original term index for each reader (for position offset calculation).
    /// term_offsets[i] = the position offset of the i-th sorted reader's
    /// term in the original phrase.
    term_offsets: Vec<u32>,
    /// Per-reader: (current_doc_id, current_positions). Reused across docs.
    reader_state: Vec<(u32, Vec<u32>)>,
    /// Current matched doc.
    current: DocId,
    idf: f32,
    avg_field_length: f32,
    /// Cached norms reader for scoring (avoids per-doc lookup).
    norms: Option<crate::inverted::norms::FieldNormsReader<'a>>,
    /// Precomputed constant score when norms are uniform AND phrase_freq=1.
    constant_score: Option<f32>,
    /// Reusable pointer array for two-pointer merge.
    ptrs_buf: Vec<usize>,
    /// Phrase frequency (count of phrase occurrences) in the current matched
    /// document. Used as TF in BM25 scoring.
    /// See [[investigation-20260405-02-phrase-tf-always-one]].
    phrase_freq: u32,
}

// SAFETY: segment_data is only dereferenced within the search call that
// holds the SegmentReader, so the pointer remains valid.
unsafe impl Send for PhraseScorer<'_> {}

impl PhraseScorer<'_> {
    /// Advance lead reader and find the next document where all terms are
    /// present with consecutive positions.
    ///
    /// See [[fix-phrase-nterm-skip-bug]].
    fn advance_to_next_phrase(&mut self) {
        let num_readers = self.readers.len();
        if num_readers == 0 {
            self.current = NO_MORE_DOCS;
            return;
        }

        if num_readers == 2 {
            self.advance_two_term_phrase();
            return;
        }

        // N-term path (3+ terms). Uses advance(current+1) for the lead
        // because next_doc() is only available on PositionPostingListReader
        // and the general TF>1 position check needs reader_state anyway.

        // Initial lead advance — advance(current+1) handles both
        // Scorer::next() and Scorer::advance(target) correctly.
        let mut lead_doc =
            match self.readers[0].advance(DocId::new(if self.current == NO_MORE_DOCS {
                0
            } else {
                self.current.as_u32() + 1
            })) {
                Some(id) => id,
                None => {
                    self.current = NO_MORE_DOCS;
                    return;
                }
            };

        loop {
            // Converge all readers onto the same document. When any reader
            // overshoots, the lead catches up and alignment restarts from
            // reader[1]. lead_doc strictly increases each round.
            'align: loop {
                let target = lead_doc.as_u32();
                let mut aligned = true;

                for i in 1..num_readers {
                    match self.readers[i].advance(lead_doc) {
                        Some(id) if id.as_u32() == target => {}
                        Some(id) => match self.readers[0].advance(id) {
                            Some(new_lead) => {
                                lead_doc = new_lead;
                                aligned = false;
                                break;
                            }
                            None => {
                                self.current = NO_MORE_DOCS;
                                return;
                            }
                        },
                        None => {
                            self.current = NO_MORE_DOCS;
                            return;
                        }
                    }
                }

                if aligned {
                    break 'align;
                }
            }

            // All readers at lead_doc — check positions and count phrase TF
            let freq = if num_readers == 1 {
                1
            } else {
                self.count_phrase_positions(lead_doc)
            };
            if freq > 0 {
                self.phrase_freq = freq;
                self.current = lead_doc;
                return;
            }

            // Position mismatch — advance lead to next doc
            self.current = lead_doc;
            lead_doc = match self.readers[0].advance(DocId::new(lead_doc.as_u32() + 1)) {
                Some(id) => id,
                None => {
                    self.current = NO_MORE_DOCS;
                    return;
                }
            };
        }
    }

    /// 2-term phrase fast path. Uses next_doc() for sequential lead
    /// iteration and an inner convergence loop for catch-up.
    ///
    /// See [[fix-phrase-nterm-skip-bug]].
    fn advance_two_term_phrase(&mut self) {
        let off0 = self.term_offsets[0];
        let off1 = self.term_offsets[1];

        // Initial lead advance uses advance(current+1) to handle
        // Scorer::advance(target) correctly. Subsequent iterations
        // use next_doc() (faster — no target comparison).
        let mut lead_doc =
            match self.readers[0].advance(DocId::new(if self.current == NO_MORE_DOCS {
                0
            } else {
                self.current.as_u32() + 1
            })) {
                Some(id) => id,
                None => {
                    self.current = NO_MORE_DOCS;
                    return;
                }
            };

        loop {
            // Converge both readers onto the same document
            loop {
                match self.readers[1].advance(lead_doc) {
                    Some(id) if id == lead_doc => break,
                    Some(id) => match self.readers[0].advance(id) {
                        Some(new_lead) => {
                            lead_doc = new_lead;
                        }
                        None => {
                            self.current = NO_MORE_DOCS;
                            return;
                        }
                    },
                    None => {
                        self.current = NO_MORE_DOCS;
                        return;
                    }
                }
            }

            // Both readers at lead_doc — check positions
            let tf0 = self.readers[0].current_tf();
            let tf1 = self.readers[1].current_tf();

            if tf0 == 1 && tf1 == 1 {
                // Both terms have TF=1 → at most one phrase occurrence.
                let pos0 = self.readers[0].first_position();
                let pos1 = self.readers[1].first_position();
                if pos0.wrapping_sub(off0) == pos1.wrapping_sub(off1) {
                    self.phrase_freq = 1;
                    self.current = lead_doc;
                    return;
                }
            } else {
                self.reader_state[0].0 = lead_doc.as_u32();
                self.reader_state[0].1.clear();
                self.reader_state[0]
                    .1
                    .extend_from_slice(self.readers[0].positions());
                self.reader_state[1].0 = lead_doc.as_u32();
                self.reader_state[1].1.clear();
                self.reader_state[1]
                    .1
                    .extend_from_slice(self.readers[1].positions());

                let freq = self.count_positions();
                if freq > 0 {
                    self.phrase_freq = freq;
                    self.current = lead_doc;
                    return;
                }
            }

            // Position mismatch — advance to next doc via next_doc()
            self.current = lead_doc;
            lead_doc = match self.readers[0].next_doc() {
                Some(id) => id,
                None => {
                    self.current = NO_MORE_DOCS;
                    return;
                }
            };
        }
    }

    /// Count phrase occurrences in the current document. Returns 0 if no
    /// match. Uses a TF=1 fast path (common case: each term appears once)
    /// without Vec copies, falling back to multi-pointer merge for
    /// high-TF documents.
    fn count_phrase_positions(&mut self, lead_doc: DocId) -> u32 {
        let num_readers = self.readers.len();

        // Fast path: all readers have TF=1 — at most one phrase occurrence.
        if self.readers.iter().all(|r| r.current_tf() == 1) {
            let base = self.readers[0]
                .first_position()
                .wrapping_sub(self.term_offsets[0]);
            let aligned = (1..num_readers).all(|i| {
                self.readers[i]
                    .first_position()
                    .wrapping_sub(self.term_offsets[i])
                    == base
            });
            return if aligned { 1 } else { 0 };
        }

        // General path: copy positions into reader_state, run multi-pointer merge
        let target = lead_doc.as_u32();
        for i in 0..num_readers {
            self.reader_state[i].0 = target;
            self.reader_state[i].1.clear();
            self.reader_state[i]
                .1
                .extend_from_slice(self.readers[i].positions());
        }
        self.count_positions()
    }

    /// Multi-pointer merge over reader_state positions. Returns the number
    /// of phrase occurrences (TF) in the current document.
    /// Uses term_offsets to account for the original phrase order
    /// (readers are sorted by doc frequency, not phrase order).
    fn count_positions(&mut self) -> u32 {
        let num = self.readers.len();
        self.ptrs_buf.clear();
        self.ptrs_buf.resize(num, 0);

        // Find the reader with the smallest term_offset to use as anchor
        let anchor = self
            .term_offsets
            .iter()
            .enumerate()
            .min_by_key(|(_, off)| *off)
            .map(|(i, _)| i)
            .unwrap();
        let anchor_offset = self.term_offsets[anchor];
        let anchor_positions = &self.reader_state[anchor].1;

        let mut count: u32 = 0;

        for &anchor_pos in anchor_positions.iter() {
            let start = anchor_pos - anchor_offset; // phrase start position
            let mut matched = true;

            for i in 0..num {
                if i == anchor {
                    continue;
                }
                let expected = start + self.term_offsets[i];
                let positions = &self.reader_state[i].1;

                while self.ptrs_buf[i] < positions.len() && positions[self.ptrs_buf[i]] < expected {
                    self.ptrs_buf[i] += 1;
                }

                if self.ptrs_buf[i] >= positions.len() || positions[self.ptrs_buf[i]] != expected {
                    matched = false;
                    break;
                }
            }

            if matched {
                count += 1;
            }
        }
        count
    }
}

impl Scorer for PhraseScorer<'_> {
    fn doc_id(&self) -> DocId {
        self.current
    }

    fn next(&mut self) -> DocId {
        self.advance_to_next_phrase();
        self.current
    }

    fn advance(&mut self, target: DocId) -> DocId {
        if self.current < target {
            self.current = DocId::new(target.as_u32().saturating_sub(1));
        }
        self.advance_to_next_phrase();
        self.current
    }

    fn score(&mut self) -> f32 {
        // The constant_score optimization assumes uniform norms AND TF=1.
        // Only use it when phrase_freq=1 (the most common case).
        if self.phrase_freq <= 1 {
            if let Some(cs) = self.constant_score {
                return cs;
            }
        }
        let dl = self
            .norms
            .as_ref()
            .map(|n| n.norm(self.doc_id()))
            .unwrap_or(1.0);
        bm25_score(self.idf, self.phrase_freq as f32, dl, self.avg_field_length)
    }

    fn two_phase(&mut self) -> Option<&mut dyn TwoPhaseIterator> {
        None
    }
}

struct SimpleIterScorer<'a> {
    postings: crate::inverted::postings::PostingListReader<'a>,
    current: DocId,
}

impl<'a> SimpleIterScorer<'a> {
    fn empty() -> Self {
        Self {
            postings: crate::inverted::postings::PostingListReader::new(&[0, 0, 0, 0, 0]),
            current: NO_MORE_DOCS,
        }
    }
}

impl Scorer for SimpleIterScorer<'_> {
    fn doc_id(&self) -> DocId {
        self.current
    }
    fn next(&mut self) -> DocId {
        self.current = match self.postings.next() {
            Some((id, _)) => id,
            None => NO_MORE_DOCS,
        };
        self.current
    }
    fn advance(&mut self, target: DocId) -> DocId {
        while self.current < target && self.current != NO_MORE_DOCS {
            self.next();
        }
        self.current
    }
    fn score(&mut self) -> f32 {
        1.0
    }
    fn two_phase(&mut self) -> Option<&mut dyn TwoPhaseIterator> {
        None
    }
}

struct BoundEmptyQuery;
impl BoundQuery for BoundEmptyQuery {
    fn scorer_supplier(&self, _: &SegmentReader) -> Result<Option<Box<dyn ScorerSupplier>>> {
        Ok(None)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::analysis::{AnalyzerRegistry, Token};
    use crate::core::SegmentId;
    use crate::mapping::{FieldType, Mapping};
    use crate::segment::builder::SegmentBuilder;

    fn make_tokens(terms: &[&str]) -> Vec<Token> {
        terms
            .iter()
            .enumerate()
            .map(|(i, t)| Token::new(*t, 0, t.len(), i as u32))
            .collect()
    }

    fn build_phrase_store() -> crate::search::segment_store::SegmentStore {
        let schema = Mapping::builder().field("body", FieldType::Text).build();
        let mut builder = SegmentBuilder::new(SegmentId::new(1), &schema);

        // Doc 0: "the quick brown fox" — positions [0,1,2,3]
        builder.add_document(
            &[(
                FieldId::new(0),
                make_tokens(&["the", "quick", "brown", "fox"]),
            )],
            br#"{"body":"the quick brown fox"}"#,
        );

        // Doc 1: "the brown quick fox" — positions [0,1,2,3] but different order
        builder.add_document(
            &[(
                FieldId::new(0),
                make_tokens(&["the", "brown", "quick", "fox"]),
            )],
            br#"{"body":"the brown quick fox"}"#,
        );

        // Doc 2: "quick fox brown" — no "the", different order
        builder.add_document(
            &[(FieldId::new(0), make_tokens(&["quick", "fox", "brown"]))],
            br#"{"body":"quick fox brown"}"#,
        );

        let reader = SegmentReader::open(builder.build()).unwrap();
        crate::search::segment_store::SegmentStore::new(
            vec![reader],
            AnalyzerRegistry::new(),
            None,
            None,
        )
    }

    #[test]
    fn phrase_exact_match() {
        let store = build_phrase_store();
        let searcher = Searcher::new(&store);
        let query = MatchPhraseQuery {
            field: "body".into(),
            query_text: "quick brown".into(),
            analyzer: None,
        };

        let weight = query.bind(&searcher, ScoreMode::Complete).unwrap();
        let supplier = weight
            .scorer_supplier(&searcher.segments()[0])
            .unwrap()
            .unwrap();
        let mut scorer = supplier.scorer().unwrap();

        // "quick brown" at consecutive positions only in doc 0
        assert_eq!(scorer.doc_id(), DocId::new(0));
        assert_eq!(scorer.next(), NO_MORE_DOCS);
    }

    #[test]
    fn phrase_wrong_order_no_match() {
        let store = build_phrase_store();
        let searcher = Searcher::new(&store);
        let query = MatchPhraseQuery {
            field: "body".into(),
            query_text: "brown quick".into(),
            analyzer: None,
        };

        let weight = query.bind(&searcher, ScoreMode::Complete).unwrap();
        let supplier = weight
            .scorer_supplier(&searcher.segments()[0])
            .unwrap()
            .unwrap();
        let mut scorer = supplier.scorer().unwrap();

        // "brown quick" is consecutive in doc 1 (positions 1,2)
        assert_eq!(scorer.doc_id(), DocId::new(1));
        assert_eq!(scorer.next(), NO_MORE_DOCS);
    }

    #[test]
    fn phrase_no_match() {
        let store = build_phrase_store();
        let searcher = Searcher::new(&store);
        let query = MatchPhraseQuery {
            field: "body".into(),
            query_text: "fox quick".into(), // never consecutive
            analyzer: None,
        };

        let weight = query.bind(&searcher, ScoreMode::Complete).unwrap();
        let supplier = weight
            .scorer_supplier(&searcher.segments()[0])
            .unwrap()
            .unwrap();
        let scorer = supplier.scorer().unwrap();

        // "fox" is always after "quick" in all docs, never before
        // Doc 0: fox@3, quick@1 → not consecutive
        // Doc 1: fox@3, quick@2 → not consecutive
        // Doc 2: fox@1, quick@0 → consecutive! "quick fox" but query is "fox quick"
        // Wait — "fox quick" means fox at position X, quick at X+1. Doc 2 has quick@0, fox@1.
        // So "fox quick" would need fox before quick at consecutive positions.
        // Doc 2: fox@1, quick@0 → fox@1 then quick@1+1=2, but quick is at 0, not 2 → no match

        assert_eq!(scorer.doc_id(), NO_MORE_DOCS);
    }

    #[test]
    fn phrase_single_term_degenerates() {
        let store = build_phrase_store();
        let searcher = Searcher::new(&store);
        let query = MatchPhraseQuery {
            field: "body".into(),
            query_text: "quick".into(),
            analyzer: None,
        };

        let weight = query.bind(&searcher, ScoreMode::Complete).unwrap();
        let supplier = weight
            .scorer_supplier(&searcher.segments()[0])
            .unwrap()
            .unwrap();
        let mut scorer = supplier.scorer().unwrap();

        // Single term → just a term query. "quick" in docs 0, 1, 2
        let mut ids = Vec::new();
        while scorer.doc_id() != NO_MORE_DOCS {
            ids.push(scorer.doc_id().as_u32());
            scorer.next();
        }
        assert_eq!(ids, vec![0, 1, 2]);
    }

    #[test]
    fn phrase_three_terms() {
        let store = build_phrase_store();
        let searcher = Searcher::new(&store);
        let query = MatchPhraseQuery {
            field: "body".into(),
            query_text: "the quick brown".into(),
            analyzer: None,
        };

        let weight = query.bind(&searcher, ScoreMode::Complete).unwrap();
        let supplier = weight
            .scorer_supplier(&searcher.segments()[0])
            .unwrap()
            .unwrap();
        let mut scorer = supplier.scorer().unwrap();

        // "the quick brown" at positions 0,1,2 only in doc 0
        assert_eq!(scorer.doc_id(), DocId::new(0));
        assert_eq!(scorer.next(), NO_MORE_DOCS);
    }

    #[test]
    fn phrase_has_positive_score() {
        let store = build_phrase_store();
        let searcher = Searcher::new(&store);
        let query = MatchPhraseQuery {
            field: "body".into(),
            query_text: "quick brown".into(),
            analyzer: None,
        };

        let weight = query.bind(&searcher, ScoreMode::Complete).unwrap();
        let supplier = weight
            .scorer_supplier(&searcher.segments()[0])
            .unwrap()
            .unwrap();
        let mut scorer = supplier.scorer().unwrap();

        assert_eq!(scorer.doc_id(), DocId::new(0));
        let score = scorer.score();
        assert!(score > 0.0, "phrase score should be positive, got {score}");
    }

    /// Regression test for [[investigation-20260405-01-phrase-2term-skip-bug]].
    ///
    /// When reader[1] overshoots and reader[0] catches up via advance(),
    /// the catch-up document must still be checked for a phrase match.
    /// The bug: next_doc() after catch-up advances past the catch-up doc.
    #[test]
    fn phrase_2term_catchup_not_skipped() {
        let schema = Mapping::builder().field("body", FieldType::Text).build();
        let mut builder = SegmentBuilder::new(SegmentId::new(1), &schema);

        // Term "alpha": docs 0, 3, 5 (3 docs — rarer, becomes reader[0])
        // Term "beta":  docs 1, 2, 3, 4, 5 (5 docs — more common, becomes reader[1])
        //
        // Iteration 1: reader[0] at doc 0, reader[1].advance(0) → doc 1 (overshoot).
        //   reader[0].advance(1) → doc 3. current = 2, continue.
        // Iteration 2 (BUG): next_doc() on reader[0] at doc 3 → doc 5. Doc 3 skipped!
        // Iteration 2 (FIX): advance(3) on reader[0] at doc 3 → doc 3 (no-op). Doc 3 checked.

        // Doc 0: has alpha, no beta
        builder.add_document(
            &[(FieldId::new(0), make_tokens(&["alpha", "gamma"]))],
            b"{}",
        );
        // Doc 1: has beta, no alpha
        builder.add_document(&[(FieldId::new(0), make_tokens(&["beta", "delta"]))], b"{}");
        // Doc 2: has beta, no alpha
        builder.add_document(
            &[(FieldId::new(0), make_tokens(&["beta", "epsilon"]))],
            b"{}",
        );
        // Doc 3: "alpha beta" — phrase match (consecutive positions)
        builder.add_document(&[(FieldId::new(0), make_tokens(&["alpha", "beta"]))], b"{}");
        // Doc 4: has beta, no alpha
        builder.add_document(&[(FieldId::new(0), make_tokens(&["beta", "zeta"]))], b"{}");
        // Doc 5: "alpha beta" — another phrase match
        builder.add_document(&[(FieldId::new(0), make_tokens(&["alpha", "beta"]))], b"{}");

        let reader = SegmentReader::open(builder.build()).unwrap();
        let store = crate::search::segment_store::SegmentStore::new(
            vec![reader],
            AnalyzerRegistry::new(),
            None,
            None,
        );
        let searcher = Searcher::new(&store);
        let query = MatchPhraseQuery {
            field: "body".into(),
            query_text: "alpha beta".into(),
            analyzer: None,
        };

        let weight = query.bind(&searcher, ScoreMode::Complete).unwrap();
        let supplier = weight
            .scorer_supplier(&searcher.segments()[0])
            .unwrap()
            .unwrap();
        let mut scorer = supplier.scorer().unwrap();

        // Both doc 3 and doc 5 must be found
        assert_eq!(
            scorer.doc_id(),
            DocId::new(3),
            "doc 3 must not be skipped after catch-up"
        );
        assert_eq!(scorer.next(), DocId::new(5));
        assert_eq!(scorer.next(), NO_MORE_DOCS);
    }

    /// Same catch-up skip bug as above, but for the N-term (3+) general path.
    ///
    /// The N-term path uses `advance(current + 1)` after catch-up, but
    /// `PositionPostingListReader::advance()` always reads the next stream
    /// entry — it does not check if `current_doc_id >= target` first.
    /// So `advance(new_lead)` when already at `new_lead` skips past it.
    #[test]
    fn phrase_nterm_catchup_not_skipped() {
        let schema = Mapping::builder().field("body", FieldType::Text).build();
        let mut builder = SegmentBuilder::new(SegmentId::new(1), &schema);

        // 3 terms forces the N-term path (2-term fast path only for exactly 2).
        //
        // Term "alpha": docs 0, 3, 5 (df=3, reader[0] after sort)
        // Term "gamma": docs 0, 3, 5 (df=3, reader[1] after sort)
        // Term "beta":  docs 1, 2, 3, 4, 5 (df=5, reader[2] after sort)
        //
        // Iteration 1 (current = NO_MORE_DOCS):
        //   reader[0] (alpha).advance(0) → doc 0
        //   reader[1] (gamma).advance(0) → doc 0. OK.
        //   reader[2] (beta).advance(0) → doc 1 (overshoot!)
        //     reader[0].advance(1) → doc 3. current = 2, continue.
        //
        // Iteration 2 (current = 2):
        //   reader[0].advance(3) — already at doc 3, but advance() reads next → doc 5!
        //   Doc 3 is SKIPPED.

        // Doc 0: alpha, gamma (no beta)
        builder.add_document(
            &[(FieldId::new(0), make_tokens(&["alpha", "gamma", "delta"]))],
            b"{}",
        );
        // Doc 1: beta only
        builder.add_document(
            &[(FieldId::new(0), make_tokens(&["beta", "delta", "epsilon"]))],
            b"{}",
        );
        // Doc 2: beta only
        builder.add_document(
            &[(FieldId::new(0), make_tokens(&["beta", "epsilon", "zeta"]))],
            b"{}",
        );
        // Doc 3: "alpha beta gamma" — phrase match
        builder.add_document(
            &[(FieldId::new(0), make_tokens(&["alpha", "beta", "gamma"]))],
            b"{}",
        );
        // Doc 4: beta only
        builder.add_document(
            &[(FieldId::new(0), make_tokens(&["beta", "eta", "theta"]))],
            b"{}",
        );
        // Doc 5: "alpha beta gamma" — another phrase match
        builder.add_document(
            &[(FieldId::new(0), make_tokens(&["alpha", "beta", "gamma"]))],
            b"{}",
        );

        let reader = SegmentReader::open(builder.build()).unwrap();
        let store = crate::search::segment_store::SegmentStore::new(
            vec![reader],
            AnalyzerRegistry::new(),
            None,
            None,
        );
        let searcher = Searcher::new(&store);
        let query = MatchPhraseQuery {
            field: "body".into(),
            query_text: "alpha beta gamma".into(),
            analyzer: None,
        };

        let weight = query.bind(&searcher, ScoreMode::Complete).unwrap();
        let supplier = weight
            .scorer_supplier(&searcher.segments()[0])
            .unwrap()
            .unwrap();
        let mut scorer = supplier.scorer().unwrap();

        // Both doc 3 and doc 5 must be found
        assert_eq!(
            scorer.doc_id(),
            DocId::new(3),
            "N-term: doc 3 must not be skipped after catch-up"
        );
        assert_eq!(scorer.next(), DocId::new(5));
        assert_eq!(scorer.next(), NO_MORE_DOCS);
    }

    /// Regression test for bug 3: `Scorer::advance(target)` must return
    /// a doc `>= target`. The 2-term path used `next_doc()` which iterates
    /// from the reader's current position, ignoring the target — it could
    /// return a match before the requested target.
    #[test]
    fn phrase_advance_respects_target() {
        let schema = Mapping::builder().field("body", FieldType::Text).build();
        let mut builder = SegmentBuilder::new(SegmentId::new(1), &schema);

        // Phrase matches at docs 0, 5, 10. Gaps filled with non-matching docs.
        // Doc 0: "alpha beta"
        builder.add_document(&[(FieldId::new(0), make_tokens(&["alpha", "beta"]))], b"{}");
        // Docs 1-4: filler (no alpha)
        for _ in 1..5 {
            builder.add_document(&[(FieldId::new(0), make_tokens(&["gamma"]))], b"{}");
        }
        // Doc 5: "alpha beta"
        builder.add_document(&[(FieldId::new(0), make_tokens(&["alpha", "beta"]))], b"{}");
        // Docs 6-9: filler
        for _ in 6..10 {
            builder.add_document(&[(FieldId::new(0), make_tokens(&["gamma"]))], b"{}");
        }
        // Doc 10: "alpha beta"
        builder.add_document(&[(FieldId::new(0), make_tokens(&["alpha", "beta"]))], b"{}");

        let reader = SegmentReader::open(builder.build()).unwrap();
        let store = crate::search::segment_store::SegmentStore::new(
            vec![reader],
            AnalyzerRegistry::new(),
            None,
            None,
        );
        let searcher = Searcher::new(&store);
        let query = MatchPhraseQuery {
            field: "body".into(),
            query_text: "alpha beta".into(),
            analyzer: None,
        };

        let weight = query.bind(&searcher, ScoreMode::Complete).unwrap();
        let supplier = weight
            .scorer_supplier(&searcher.segments()[0])
            .unwrap()
            .unwrap();
        let mut scorer = supplier.scorer().unwrap();

        // Initial match at doc 0
        assert_eq!(scorer.doc_id(), DocId::new(0));

        // advance(7) must return >= 7 — must NOT return doc 5
        let result = scorer.advance(DocId::new(7));
        assert!(
            result >= DocId::new(7),
            "advance(7) returned {result:?}, expected >= DocId(7)"
        );
        assert_eq!(result, DocId::new(10));
    }

    /// Regression test for [[investigation-20260405-02-phrase-tf-always-one]].
    ///
    /// Phrase scorer hardcoded TF=1.0 in BM25, so a doc with the phrase
    /// repeated multiple times scored the same as a doc with one
    /// occurrence (per-token-length terms equal). The fix counts phrase
    /// occurrences and uses that as TF.
    #[test]
    fn phrase_freq_repeated_phrase_scores_higher() {
        let schema = Mapping::builder().field("body", FieldType::Text).build();
        let mut builder = SegmentBuilder::new(SegmentId::new(1), &schema);

        // Doc 0: phrase "alpha beta" appears 3 times
        // body length = 6 tokens
        builder.add_document(
            &[(
                FieldId::new(0),
                make_tokens(&["alpha", "beta", "alpha", "beta", "alpha", "beta"]),
            )],
            b"{}",
        );
        // Doc 1: phrase "alpha beta" appears 1 time
        // body length = 6 tokens (same as doc 0 — eliminates dl-norm differences)
        builder.add_document(
            &[(
                FieldId::new(0),
                make_tokens(&["alpha", "beta", "gamma", "delta", "epsilon", "zeta"]),
            )],
            b"{}",
        );

        let reader = SegmentReader::open(builder.build()).unwrap();
        let store = crate::search::segment_store::SegmentStore::new(
            vec![reader],
            AnalyzerRegistry::new(),
            None,
            None,
        );
        let searcher = Searcher::new(&store);
        let query = MatchPhraseQuery {
            field: "body".into(),
            query_text: "alpha beta".into(),
            analyzer: None,
        };

        let weight = query.bind(&searcher, ScoreMode::Complete).unwrap();
        let supplier = weight
            .scorer_supplier(&searcher.segments()[0])
            .unwrap()
            .unwrap();
        let mut scorer = supplier.scorer().unwrap();

        // Doc 0 has phrase 3 times, doc 1 has it 1 time
        assert_eq!(scorer.doc_id(), DocId::new(0));
        let doc0_score = scorer.score();
        scorer.next();
        assert_eq!(scorer.doc_id(), DocId::new(1));
        let doc1_score = scorer.score();

        // BM25 with higher TF should produce a strictly higher score.
        // Both docs have the same field length, so the only score
        // difference comes from TF.
        assert!(
            doc0_score > doc1_score,
            "doc with 3 phrase occurrences ({doc0_score}) must score higher than \
             doc with 1 occurrence ({doc1_score}) — phrase TF must be counted"
        );
    }
}