use crate::core::analysis::analyzer::{Analyzer, AnalyzerStoredValue, TokenStreamComponents};
use crate::core::analysis::reader::ReaderEnum;
use crate::core::analysis::token_stream::TokenStream;
use crate::core::document::document::Document;
use crate::core::document::field::Store;
use crate::core::document::fields::FieldTokenStreamEnum;
use crate::core::document::text_field::TextField;
use crate::core::index::bytes_ref::BytesRefValueEnum;
use crate::core::index::impact::Impact;
use crate::core::index::impacts::Impacts;
use crate::core::index::impacts_enum::ImpactsEnum;
use crate::core::index::impacts_source::ImpactsSource;
use crate::core::index::index_reader::IndexReader;
use crate::core::index::live_index_writer_config::LiveIndexWriterConfig;
use crate::core::index::postings_enum::PostingsEnum;
use crate::core::index::term::Term;
use crate::core::search::boolean_clause::Occur;
use crate::core::search::boolean_query::Builder;
use crate::core::search::doc_id_set_iterator::DocIdSetIterator;
use crate::core::search::exact_phrase_matcher::merge_impacts_from_ie;
use crate::core::search::phrase_query::PhraseQuery;
use crate::core::search::query::{Query, QueryBase};
use crate::core::search::score_doc::ScoreDocLike;
use crate::core::search::similarities_impl::bm25_similarity::BM25Similarity;
use crate::core::search::similarities_impl::classic_similarity;
use crate::core::search::term_query::TermQuery;
use crate::core::search::top_docs::TopDocsLike;
use crate::core::search::top_score_doc_collector_manager::TopScoreDocCollectorManager;
use crate::core::util::attribute_source::AttributeSource;
use crate::core::util::close::{Closeable, CloseableRef};
use crate::core::util::error::lucene_error::{LuceneError, Result};
use crate::core::util::io_utils::IOUtils;
use crate::test_framework::core::analysis::mock_analyzer::{MockAnalyzer, WHITESPACE};
use crate::test_framework::core::analysis::mock_token_filter::ENGLISH_STOPSET;
use crate::test_framework::core::analysis::mock_tokenizer::{MockTokenizer, SIMPLE};
use crate::test_framework::core::analysis::{canned_token_stream::CannedTokenStream, token};
use crate::test_framework::core::index::random_index_writer::RandomIndexWriter;
use crate::test_framework::core::search::check_hits::CheckHits;
use crate::test_framework::core::search::query_utils::QueryUtils;
use crate::test_framework::core::util::DefaultIndexSearchCR;
use crate::test_framework::core::util::lucene_test_case::{
at_least, at_least_usize, is_night_mode, new_directory_shared, new_index_writer_config,
new_index_writer_config_with_analyzer, new_log_merge_policy, new_searcher_with_reader,
new_text_field, random, random_from_seed,
};
use crate::test_framework::core::util::test_util::TestUtil;
use rand::prelude::SliceRandom;
use rand::{Rng, RngExt};
use std::collections::HashMap;
use std::rc::Rc;
use std::sync::LazyLock;
#[allow(dead_code)]
struct TestPhraseQuery;
pub const SCORE_COMP_THRESH: f32 = 1e-6;
static CONTEXT: LazyLock<DefaultIndexSearchCR> = LazyLock::new(|| {
let mut random = random();
set_up(&mut random).expect("failed to initialize TestPhraseQuery")
});
struct PhraseQueryAnalyzer {
stored_value: AnalyzerStoredValue,
seed: u64,
}
impl Analyzer for PhraseQueryAnalyzer {
fn create_components(&self, _field_name: &str) -> Result<TokenStreamComponents> {
let tokenizer = MockTokenizer::with_default_max_token_length(
random_from_seed(self.seed),
WHITESPACE.clone(),
false,
);
Ok(TokenStreamComponents::new(
Box::new(tokenizer) as Box<dyn TokenStream + Send + Sync>,
None,
))
}
fn get_position_increment_gap(&self, _field_name: &str) -> i32 {
100
}
fn stored_value(&self) -> &AnalyzerStoredValue {
&self.stored_value
}
}
crate::impl_analyzer_close!(PhraseQueryAnalyzer);
fn set_up<R>(random: &mut R) -> Result<DefaultIndexSearchCR>
where
R: Rng + ?Sized,
{
let dir = new_directory_shared(random)?;
let analyzer = PhraseQueryAnalyzer {
stored_value: AnalyzerStoredValue::new(),
seed: random.random(),
};
let writer =
RandomIndexWriter::with_analyzer(random, dir.clone(), Box::new(analyzer) as Box<dyn Analyzer>)?;
let mut field_to_type = HashMap::new();
let mut doc = Document::new();
doc.add(new_text_field(
random,
"field",
"one two three four five",
Store::Yes,
&mut field_to_type,
)?);
doc.add(new_text_field(
random,
"repeated",
"this is a repeated field - first part",
Store::Yes,
&mut field_to_type,
)?);
let repeated_field = new_text_field(
random,
"repeated",
"second part of a repeated field",
Store::Yes,
&mut field_to_type,
)?;
doc.add(repeated_field);
doc.add(new_text_field(
random,
"palindrome",
"one two three two one",
Store::Yes,
&mut field_to_type,
)?);
writer.add_document(random, doc)?;
let mut doc = Document::new();
doc.add(new_text_field(
random,
"nonexist",
"phrase exist notexist exist found",
Store::Yes,
&mut field_to_type,
)?);
writer.add_document(random, doc)?;
let mut doc = Document::new();
doc.add(new_text_field(
random,
"nonexist",
"phrase exist notexist exist found",
Store::Yes,
&mut field_to_type,
)?);
writer.add_document(random, doc)?;
let reader = writer.get_reader(random)?;
writer.close(random)?;
let searcher = new_searcher_with_reader(reader)?;
Ok(searcher)
}
#[test]
fn test_not_close_enough() -> Result<()> {
let mut random = random();
let searcher = &*CONTEXT;
let query = PhraseQuery::from_terms(2, "field", ["one", "five"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(0, hits.len());
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
Ok(())
}
#[test]
fn test_barely_close_enough() -> Result<()> {
let mut random = random();
let searcher = &*CONTEXT;
let query = PhraseQuery::from_terms(3, "field", ["one", "five"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len());
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
Ok(())
}
#[test]
fn test_exact() -> Result<()> {
let mut random = random();
let searcher = &*CONTEXT;
let query = PhraseQuery::from_terms(0, "field", ["four", "five"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "exact match");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(0, "field", ["two", "one"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(0, hits.len(), "reverse not exact");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
Ok(())
}
#[test]
fn test_slop1() -> Result<()> {
let mut random = random();
let searcher = &*CONTEXT;
let query = PhraseQuery::from_terms(1, "field", ["one", "two"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "in order");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(1, "field", ["two", "one"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(0, hits.len(), "reversed, slop not 2 or more");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
Ok(())
}
#[test]
fn test_order_doesnt_matter() -> Result<()> {
let mut random = random();
let searcher = &*CONTEXT;
let query = PhraseQuery::from_terms(2, "field", ["two", "one"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "just sloppy enough");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(2, "field", ["three", "one"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(0, hits.len(), "not sloppy enough");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
Ok(())
}
#[test]
fn test_multiple_terms() -> Result<()> {
let mut random = random();
let searcher = &*CONTEXT;
let query = PhraseQuery::from_terms(2, "field", ["one", "three", "five"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "two total moves");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(5, "field", ["five", "three", "one"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(0, hits.len(), "slop of 5 not close enough");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(6, "field", ["five", "three", "one"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "slop of 6 just right");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
Ok(())
}
#[test]
fn test_phrase_query_with_stop_analyzer() -> Result<()> {
let mut random = random();
let dir = new_directory_shared(&mut random)?;
let stop_analyzer =
MockAnalyzer::with_filter(&mut random, SIMPLE.clone(), true, ENGLISH_STOPSET.clone());
let writer = RandomIndexWriter::with_analyzer(&mut random, dir.clone(), stop_analyzer)?;
let mut field_to_type = HashMap::new();
let mut doc = Document::new();
doc.add(new_text_field(
&mut random,
"field",
"the stop words are here",
Store::Yes,
&mut field_to_type,
)?);
writer.add_document(&mut random, doc)?;
let reader = writer.get_reader(&mut random)?;
writer.close(&mut random)?;
let searcher = new_searcher_with_reader(reader)?;
let query = PhraseQuery::from_terms(0, "field", ["stop", "words"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len());
QueryUtils::check_from_searcher(&mut random, query, &searcher)?;
Ok(())
}
#[test]
fn test_phrase_query_in_conjunction_scorer() -> Result<()> {
let mut random = random();
let dir = new_directory_shared(&mut random)?;
let mut field_to_type = HashMap::new();
{
let writer = RandomIndexWriter::new(&mut random, dir.clone())?;
let mut doc = Document::new();
doc.add(new_text_field(
&mut random,
"source",
"marketing info",
Store::Yes,
&mut field_to_type,
)?);
writer.add_document(&mut random, doc)?;
let mut doc = Document::new();
doc.add(new_text_field(
&mut random,
"contents",
"foobar",
Store::Yes,
&mut field_to_type,
)?);
doc.add(new_text_field(
&mut random,
"source",
"marketing info",
Store::Yes,
&mut field_to_type,
)?);
writer.add_document(&mut random, doc)?;
let reader = writer.get_reader(&mut random)?;
writer.close(&mut random)?;
let searcher = new_searcher_with_reader(reader)?;
let phrase_query = PhraseQuery::from_terms(0, "source", ["marketing", "info"])?;
let top_docs = searcher.search(phrase_query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(2, hits.len());
QueryUtils::check_from_searcher(&mut random, phrase_query.clone(), &searcher)?;
let term_query: Query = TermQuery::new(Term::from_text("contents", "foobar")).into();
let mut b = Builder::new();
b.add(term_query.clone(), Occur::Must)?;
b.add(phrase_query.clone(), Occur::Must)?;
let boolean_query: Query = b.build().into();
let top_docs = searcher.search(boolean_query, 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len());
QueryUtils::check_from_searcher(&mut random, term_query, &searcher)?;
}
{
let writer = RandomIndexWriter::new(&mut random, dir.clone())?;
let mut doc = Document::new();
doc.add(new_text_field(
&mut random,
"contents",
"map entry woo",
Store::Yes,
&mut field_to_type,
)?);
writer.add_document(&mut random, doc)?;
let mut doc = Document::new();
doc.add(new_text_field(
&mut random,
"contents",
"woo map entry",
Store::Yes,
&mut field_to_type,
)?);
writer.add_document(&mut random, doc)?;
let mut doc = Document::new();
doc.add(new_text_field(
&mut random,
"contents",
"map foobarword entry woo",
Store::Yes,
&mut field_to_type,
)?);
writer.add_document(&mut random, doc)?;
let reader = writer.get_reader(&mut random)?;
writer.close(&mut random)?;
let searcher = new_searcher_with_reader(reader)?;
let term_query: Query = TermQuery::new(Term::from_text("contents", "woo")).into();
let phrase_query = PhraseQuery::from_terms(0, "contents", ["map", "entry"])?;
let top_docs = searcher.search(term_query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(3, hits.len());
let top_docs = searcher.search(phrase_query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(2, hits.len());
let mut b = Builder::new();
b.add(term_query.clone(), Occur::Must)?;
b.add(phrase_query.clone(), Occur::Must)?;
let boolean_query1: Query = b.build().into();
let top_docs = searcher.search(boolean_query1, 1000)?;
let hits = top_docs.score_docs();
assert_eq!(2, hits.len());
let mut b = Builder::new();
b.add(phrase_query, Occur::Must)?;
b.add(term_query, Occur::Must)?;
let boolean_query2: Query = b.build().into();
let top_docs = searcher.search(boolean_query2.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(2, hits.len());
QueryUtils::check_from_searcher(&mut random, boolean_query2, &searcher)?;
}
Ok(())
}
#[test]
fn test_slop_scoring() -> Result<()> {
let mut random = random();
let dir = new_directory_shared(&mut random)?;
let mut iwc = new_index_writer_config(&mut random)?;
iwc.set_merge_policy(new_log_merge_policy(&mut random)?);
let writer = RandomIndexWriter::with_config(&mut random, dir.clone(), iwc);
let mut field_to_type = HashMap::new();
let mut doc = Document::new();
doc.add(new_text_field(
&mut random,
"field",
"foo firstname lastname foo",
Store::Yes,
&mut field_to_type,
)?);
writer.add_document(&mut random, doc)?;
let mut doc2 = Document::new();
doc2.add(new_text_field(
&mut random,
"field",
"foo firstname zzz lastname foo",
Store::Yes,
&mut field_to_type,
)?);
writer.add_document(&mut random, doc2)?;
let mut doc3 = Document::new();
doc3.add(new_text_field(
&mut random,
"field",
"foo firstname zzz yyy lastname foo",
Store::Yes,
&mut field_to_type,
)?);
writer.add_document(&mut random, doc3)?;
let reader = writer.get_reader(&mut random)?;
writer.close(&mut random)?;
let searcher =
crate::test_framework::core::util::lucene_test_case::new_searcher_with_reader_and_similarity(
reader,
std::sync::Arc::new(classic_similarity::new()),
)?;
let query = PhraseQuery::from_terms(i32::MAX as usize, "field", ["firstname", "lastname"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(3, hits.len());
assert!((hits[0].score - 1.0).abs() <= 0.01);
assert_eq!(0, hits[0].doc);
assert!((hits[1].score - 0.63).abs() <= 0.01);
assert_eq!(1, hits[1].doc);
assert!((hits[2].score - 0.47).abs() <= 0.01);
assert_eq!(2, hits[2].doc);
QueryUtils::check_from_searcher(&mut random, query, &searcher)?;
Ok(())
}
#[test]
fn test_to_string() -> Result<()> {
let q = PhraseQuery::from_terms(0, "field", std::iter::empty::<&str>())?;
assert_eq!("\"\"", q.to_string("")?);
let mut builder = crate::core::search::phrase_query::Builder::new();
builder.add(Term::from_text("field", "hi"), 1)?;
let q = builder.build()?;
assert_eq!("field:\"? hi\"", q.to_string("")?);
let mut builder = crate::core::search::phrase_query::Builder::new();
builder.add(Term::from_text("field", "hi"), 1)?;
builder.add(Term::from_text("field", "test"), 5)?;
let q = builder.build()?;
assert_eq!("field:\"? hi ? ? ? test\"", q.to_string("")?);
let mut builder = crate::core::search::phrase_query::Builder::new();
builder.add(Term::from_text("field", "hi"), 1)?;
builder.add(Term::from_text("field", "hello"), 1)?;
builder.add(Term::from_text("field", "test"), 5)?;
let q = builder.build()?;
assert_eq!("field:\"? hi|hello ? ? ? test\"", q.to_string("")?);
let mut builder = crate::core::search::phrase_query::Builder::new();
builder.add(Term::from_text("field", "hi"), 1)?;
builder.add(Term::from_text("field", "hello"), 1)?;
builder.add(Term::from_text("field", "test"), 5)?;
builder.set_slop(5);
let q = builder.build()?;
assert_eq!("field:\"? hi|hello ? ? ? test\"~5", q.to_string("")?);
Ok(())
}
#[test]
fn test_wrapped_phrase() -> Result<()> {
let mut random = random();
let searcher = &*CONTEXT;
let query = PhraseQuery::from_terms(100, "repeated", ["first", "part", "second", "part"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "slop of 100 just right");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(99, "repeated", ["first", "part", "second", "part"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(0, hits.len(), "slop of 99 not enough");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
Ok(())
}
#[test]
fn test_non_existing_phrase() -> Result<()> {
let mut random = random();
let searcher = &*CONTEXT;
let query = PhraseQuery::from_terms(2, "nonexist", ["phrase", "notexist", "found"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(2, hits.len(), "phrase without repetitions exists in 2 docs");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(1, "nonexist", ["phrase", "exist", "exist"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(2, hits.len(), "phrase with repetitions exists in two docs");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(1000, "nonexist", ["phrase", "notexist", "phrase"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(
0,
hits.len(),
"nonexisting phrase with repetitions does not exist in any doc"
);
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(1000, "nonexist", ["phrase", "exist", "exist", "exist"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(
0,
hits.len(),
"nonexisting phrase with repetitions does not exist in any doc"
);
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
Ok(())
}
#[test]
fn test_palyndrome2() -> Result<()> {
let mut random = random();
let searcher = &*CONTEXT;
let query = PhraseQuery::from_terms(0, "field", ["two", "three"])?; let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "phrase found with exact phrase scorer");
let score0 = hits[0].score;
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(2, "field", ["two", "three"])?; let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "just sloppy enough");
let score1 = hits[0].score;
assert!(
(score0 - score1).abs() <= SCORE_COMP_THRESH,
"exact scorer and sloppy scorer score the same when slop does not matter"
);
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(2, "palindrome", ["two", "three"])?; let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "just sloppy enough");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(2, "palindrome", ["three", "two"])?; let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "just sloppy enough");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
Ok(())
}
#[test]
fn test_palyndrome3() -> Result<()> {
let mut random = random();
let searcher = &*CONTEXT;
let query = PhraseQuery::from_terms(0, "field", ["one", "two", "three"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "phrase found with exact phrase scorer");
let score0 = hits[0].score;
QueryUtils::check_from_searcher(&mut random, query.clone(), searcher)?;
searcher.explain(query, 0)?;
let query = PhraseQuery::from_terms(4, "field", ["one", "two", "three"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "just sloppy enough");
let score1 = hits[0].score;
assert!(
(score0 - score1).abs() <= SCORE_COMP_THRESH,
"exact scorer and sloppy scorer score the same when slop does not matter"
);
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(4, "palindrome", ["one", "two", "three"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
let _ = searcher.explain(query.clone(), 0)?;
assert_eq!(1, hits.len(), "just sloppy enough");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
let query = PhraseQuery::from_terms(4, "palindrome", ["three", "two", "one"])?;
let top_docs = searcher.search(query.clone(), 1000)?;
let hits = top_docs.score_docs();
assert_eq!(1, hits.len(), "just sloppy enough");
QueryUtils::check_from_searcher(&mut random, query, searcher)?;
Ok(())
}
#[test]
fn test_empty_phrase_query() -> Result<()> {
let mut b = Builder::new();
b.add(
PhraseQuery::from_terms(0, "field", std::iter::empty::<&str>())?,
Occur::Must,
)?;
let q: Query = b.build().into();
let _ = q.to_string("");
Ok(())
}
#[test]
fn test_rewrite() -> Result<()> {
let searcher = &*CONTEXT;
let pq: Query = PhraseQuery::from_terms(0, "foo", ["bar"])?.into();
let rewritten = pq.rewrite(searcher)?;
assert!(matches!(rewritten, Some(Query::Term(_))));
Ok(())
}
#[test]
fn test_zero_pos_incr() -> Result<()> {
let mut random = random();
let dir = new_directory_shared(&mut random)?;
let tokens = vec![
token::with_pos_inc("a", 1, 0, 1)?,
token::with_pos_inc("aa", 0, 0, 2)?,
token::with_pos_inc("b", 1, 3, 4)?,
];
let writer = RandomIndexWriter::new(&mut random, dir.clone())?;
let mut doc = Document::new();
doc.add(TextField::from_token_stream(
"field",
FieldTokenStreamEnum::custom(CannedTokenStream::new(tokens)),
)?);
writer.add_document(&mut random, doc)?;
let reader = writer.get_reader(&mut random)?;
writer.close(&mut random)?;
let searcher = new_searcher_with_reader(reader)?;
let mut pq_builder = crate::core::search::phrase_query::Builder::new();
pq_builder.add(Term::from_text("field", "a"), 0)?;
pq_builder.add(Term::from_text("field", "b"), 1)?;
assert_eq!(1, searcher.count(pq_builder.build()?)?);
let mut pq_builder = crate::core::search::phrase_query::Builder::new();
pq_builder.add(Term::from_text("field", "a"), 0)?;
pq_builder.add(Term::from_text("field", "aa"), 0)?;
pq_builder.add(Term::from_text("field", "b"), 1)?;
assert_eq!(1, searcher.count(pq_builder.build()?)?);
let mut pq_builder = crate::core::search::phrase_query::Builder::new();
pq_builder.add(Term::from_text("field", "a"), 0)?;
pq_builder.add(Term::from_text("field", "z"), 0)?;
pq_builder.add(Term::from_text("field", "b"), 1)?;
assert_eq!(0, searcher.count(pq_builder.build()?)?);
Ok(())
}
#[test]
fn test_random_phrases() -> Result<()> {
let mut random = random();
let dir = new_directory_shared(&mut random)?;
let analyzer = MockAnalyzer::new(&mut random);
let mut iwc = new_index_writer_config_with_analyzer(&mut random, analyzer)?;
iwc.set_merge_policy(new_log_merge_policy(&mut random)?);
let writer = RandomIndexWriter::with_config(&mut random, dir.clone(), iwc);
let mut docs: Vec<Vec<String>> = Vec::new();
let mut field_to_type = HashMap::new();
let num_docs = at_least_usize(&mut random, 10);
docs.reserve(num_docs);
let mut document = Document::new();
for _ in 0..num_docs {
let term_count = if is_night_mode() {
at_least_usize(&mut random, 4097)
} else {
at_least_usize(&mut random, 200)
};
let mut doc_terms = Vec::with_capacity(term_count);
let mut text = String::new();
while doc_terms.len() < term_count {
if random.random_range(0..5) == 1 || docs.is_empty() {
let term = loop {
let term = TestUtil::random_unicode_string(&mut random);
if !term.is_empty() {
break term;
}
};
let mut ts = writer
.w
.get_config()?
.get_analyzer()
.token_stream("ignore", ReaderEnum::from(&term))?;
let body_result = (|| -> Result<()> {
ts.reset()?;
while ts.increment_token()? {
let term_bytes = ts
.get_attribute_source_mut()
.get_bytes_ref()?
.ok_or_else(|| LuceneError::illegal_state("term bytes are missing"))?;
let token = term_bytes.utf8_to_string()?;
doc_terms.push(token);
text.push_str(&doc_terms[doc_terms.len() - 1]);
text.push(' ');
}
ts.end()
})();
IOUtils::use_or_suppress_result(body_result, ts.close())?;
} else {
let last_doc = &docs[random.random_range(0..docs.len())];
let len = TestUtil::next_usize(&mut random, 1, 10);
let start = random.random_range(0..last_doc.len() - len);
for term in &last_doc[start..start + len] {
doc_terms.push(term.clone());
text.push_str(term);
text.push(' ');
}
}
}
document.clear();
document.add(new_text_field(
&mut random,
"f",
text,
Store::No,
&mut field_to_type,
)?);
writer.add_document(&mut random, &mut document)?;
docs.push(doc_terms);
}
let reader = writer.get_reader(&mut random)?;
let searcher = new_searcher_with_reader(reader)?;
writer.close(&mut random)?;
let num = at_least(&mut random, 3);
for i in 0..num {
let doc_id = random.random_range(0..docs.len());
let doc = &docs[doc_id];
let num_terms = TestUtil::next_usize(&mut random, 2, 20);
let start = random.random_range(0..doc.len() - num_terms);
let mut builder = crate::core::search::phrase_query::Builder::new();
let mut phrase = String::new();
for (position, term) in doc.iter().enumerate().skip(start).take(num_terms) {
builder.add(Term::from_text("f", term), position)?;
phrase.push_str(term);
phrase.push(' ');
}
let hits = searcher.search(builder.build()?, num_docs)?;
assert!(
hits
.score_docs()
.iter()
.any(|hit| hit.doc() == doc_id as i32),
"phrase '{}' not found; start={}, it={}, expected doc {}",
phrase,
start,
i,
doc_id
);
}
searcher.get_index_reader().close()?;
dir.as_ref().close()?;
Ok(())
}
#[test]
#[ignore = "Java-only: Rust represents phrase slop as usize, so a negative slop cannot be supplied"]
fn test_negative_slop() -> Result<()> {
test_not_required_in_rust_lucene!();
}
#[test]
#[ignore = "Java-only: Rust represents phrase positions as usize, so a negative position cannot be supplied"]
fn test_negative_position() -> Result<()> {
test_not_required_in_rust_lucene!();
}
#[test]
fn test_backward_positions() -> Result<()> {
let mut builder = crate::core::search::phrase_query::Builder::new();
builder.add(Term::from_text("field", "one"), 1)?;
builder.add(Term::from_text("field", "two"), 5)?;
let result = builder.add(Term::from_text("field", "three"), 4);
assert!(result.is_err());
Ok(())
}
static DOCS: [&str; 6] = [
"a b c d e f g h",
"b c b",
"c d d d e f g b",
"c b a b c",
"a a b b c c d d",
"a b c d a b c d a b c d",
];
#[test]
fn test_top_phrases() -> Result<()> {
let mut random = random();
let dir = new_directory_shared(&mut random)?;
let writer = RandomIndexWriter::new(&mut random, dir.clone())?;
let mut field_to_type = HashMap::new();
let mut docs = DOCS;
docs.shuffle(&mut random);
for value in docs {
let mut doc = Document::new();
doc.add(new_text_field(
&mut random,
"f",
value,
Store::No,
&mut field_to_type,
)?);
writer.add_document(&mut random, doc)?;
}
let reader = writer.get_reader(&mut random)?;
writer.close(&mut random)?;
let searcher = new_searcher_with_reader(reader)?;
let queries: Vec<Query> = vec![
PhraseQuery::from_terms(0, "f", ["b", "c"])?.into(), PhraseQuery::from_terms(0, "f", ["e", "f"])?.into(), PhraseQuery::from_terms(0, "f", ["d", "d"])?.into(), ];
for query in queries {
for top_n in 1..=2 {
let collector_manager = TopScoreDocCollectorManager::new(top_n, i32::MAX as usize)?;
let top_docs1 = searcher.search_with_collector_manager(query.clone(), &collector_manager)?;
let hits1 = top_docs1.score_docs();
let collector_manager = TopScoreDocCollectorManager::new(top_n, 1)?;
let top_docs2 = searcher.search_with_collector_manager(query.clone(), &collector_manager)?;
let hits2 = top_docs2.score_docs();
assert!(!hits1.is_empty(), "{}", query.to_string("")?);
CheckHits::check_equal(&query, hits1, hits2)?;
}
}
Ok(())
}
#[test]
fn test_merge_impacts() -> Result<()> {
let impacts1 = DummyImpactsEnum::new(1000);
let impacts2 = DummyImpactsEnum::new(2000);
let mut merged_impacts = merge_impacts_from_ie(vec![impacts1, impacts2])?;
merged_impacts.impacts_enums.iterator_at_mut(0).reset(
vec![
vec![Impact::new(3, 10), Impact::new(5, 12), Impact::new(8, 13)],
vec![
Impact::new(3, 10),
Impact::new(5, 11),
Impact::new(8, 13),
Impact::new(12, 14),
],
],
vec![110, 945],
);
merged_impacts
.impacts_enums
.iterator_at_mut(1)
.reset(vec![], vec![]);
assert_impacts_eq(
&[
&[Impact::new(3, 10), Impact::new(5, 12), Impact::new(8, 13)],
&[
Impact::new(3, 10),
Impact::new(5, 11),
Impact::new(8, 13),
Impact::new(12, 14),
],
],
&[110, 945],
&merged_impacts.get_impacts()?,
)?;
merged_impacts
.impacts_enums
.iterator_at_mut(1)
.reset(vec![vec![Impact::new(i32::MAX, 1)]], vec![5000]);
assert_impacts_eq(
&[
&[Impact::new(3, 10), Impact::new(5, 12), Impact::new(8, 13)],
&[
Impact::new(3, 10),
Impact::new(5, 11),
Impact::new(8, 13),
Impact::new(12, 14),
],
],
&[110, 945],
&merged_impacts.get_impacts()?,
)?;
merged_impacts
.impacts_enums
.iterator_at_mut(1)
.reset(vec![vec![Impact::new(i32::MAX, 2)]], vec![5000]);
assert_impacts_eq(
&[
&[Impact::new(3, 10), Impact::new(5, 12), Impact::new(8, 13)],
&[
Impact::new(3, 10),
Impact::new(5, 11),
Impact::new(8, 13),
Impact::new(12, 14),
],
],
&[110, 945],
&merged_impacts.get_impacts()?,
)?;
merged_impacts.impacts_enums.iterator_at_mut(1).reset(
vec![
vec![Impact::new(2, 10), Impact::new(6, 13)],
vec![Impact::new(3, 9), Impact::new(5, 11), Impact::new(7, 13)],
],
vec![90, 1000],
);
assert_impacts_eq(
&[
&[Impact::new(3, 10), Impact::new(5, 12), Impact::new(7, 13)],
&[Impact::new(3, 10), Impact::new(5, 11), Impact::new(7, 13)],
],
&[110, 945],
&merged_impacts.get_impacts()?,
)?;
merged_impacts.impacts_enums.iterator_at_mut(1).reset(
vec![
vec![Impact::new(2, 10), Impact::new(6, 11)],
vec![Impact::new(3, 9), Impact::new(5, 11), Impact::new(7, 13)],
],
vec![150, 900],
);
assert_impacts_eq(
&[
&[
Impact::new(2, 10),
Impact::new(3, 11),
Impact::new(5, 12),
Impact::new(6, 13),
],
&[
Impact::new(3, 10),
Impact::new(5, 11),
Impact::new(8, 13),
Impact::new(12, 14),
],
],
&[110, 945],
&merged_impacts.get_impacts()?,
)?;
merged_impacts.impacts_enums.iterator_at_mut(1).reset(
vec![
vec![Impact::new(4, 10), Impact::new(9, 13)],
vec![
Impact::new(1, 1),
Impact::new(4, 10),
Impact::new(5, 11),
Impact::new(8, 13),
Impact::new(12, 14),
Impact::new(13, 15),
],
],
vec![113, 950],
);
assert_impacts_eq(
&[
&[Impact::new(3, 10), Impact::new(4, 12), Impact::new(8, 13)],
&[
Impact::new(3, 10),
Impact::new(5, 11),
Impact::new(8, 13),
Impact::new(12, 14),
],
],
&[110, 945],
&merged_impacts.get_impacts()?,
)?;
merged_impacts.impacts_enums.iterator_at_mut(0).reset(
vec![
vec![Impact::new(3, 10), Impact::new(5, -10), Impact::new(8, -5)],
vec![
Impact::new(3, 10),
Impact::new(5, -15),
Impact::new(8, -5),
Impact::new(12, -3),
],
],
vec![110, 945],
);
merged_impacts.impacts_enums.iterator_at_mut(1).reset(
vec![
vec![Impact::new(2, 10), Impact::new(12, -4)],
vec![Impact::new(3, 9), Impact::new(12, -4), Impact::new(20, -1)],
],
vec![150, 960],
);
assert_impacts_eq(
&[
&[Impact::new(2, 10), Impact::new(8, -4)],
&[Impact::new(3, 10), Impact::new(8, -4), Impact::new(12, -3)],
],
&[110, 945],
&merged_impacts.get_impacts()?,
)?;
Ok(())
}
fn assert_impacts_eq<T>(impacts: &[&[Impact]], doc_id_upto: &[i32], actual: &T) -> Result<()>
where
T: Impacts,
{
assert_eq!(impacts.len(), actual.num_levels());
for i in 0..impacts.len() {
assert_eq!(doc_id_upto[i], actual.get_doc_id_upto(i));
let actual_impacts = actual.get_impacts(i)?;
let expect = impacts[i];
assert_eq!(expect, actual_impacts.as_slice());
}
Ok(())
}
struct DummyImpactsEnum {
cost: i64,
impacts: Rc<Vec<Vec<Impact>>>,
doc_id_upto: Rc<Vec<i32>>,
}
impl DummyImpactsEnum {
fn new(cost: i64) -> Self {
Self {
cost,
impacts: Rc::new(vec![vec![]]),
doc_id_upto: Rc::new(vec![]),
}
}
fn reset(&mut self, impacts: Vec<Vec<Impact>>, doc_id_upto: Vec<i32>) {
self.impacts = Rc::new(impacts);
self.doc_id_upto = Rc::new(doc_id_upto);
}
}
impl PostingsEnum for DummyImpactsEnum {
fn freq(&mut self) -> Result<i32> {
Err(LuceneError::unsupported_operation(""))
}
fn next_position(&mut self) -> Result<i32> {
Err(LuceneError::unsupported_operation(""))
}
fn start_offset(&self) -> Result<i32> {
Err(LuceneError::unsupported_operation(""))
}
fn end_offset(&self) -> Result<i32> {
Err(LuceneError::unsupported_operation(""))
}
fn get_payload(&self) -> Result<Option<BytesRefValueEnum<'_>>> {
Err(LuceneError::unsupported_operation(""))
}
}
impl crate::core::search::doc_id_set_iterator::DocIdSetIteratorExtensions for DummyImpactsEnum {}
impl crate::core::search::doc_id_set_iterator::BitSetIteratorAccess for DummyImpactsEnum {}
impl DocIdSetIterator for DummyImpactsEnum {
fn doc_id(&self) -> i32 {
unreachable!("")
}
fn next_doc(&mut self) -> Result<i32> {
Err(LuceneError::unsupported_operation(""))
}
fn advance(&mut self, _target: i32) -> Result<i32> {
Err(LuceneError::unsupported_operation(""))
}
fn cost(&self) -> Result<i64> {
Ok(self.cost)
}
}
impl ImpactsSource for DummyImpactsEnum {
fn advance_shallow(&mut self, _target: i32) -> Result<()> {
Err(LuceneError::unsupported_operation(""))
}
type Impacts<'a>
= ImpactsImpl
where
Self: 'a;
fn get_impacts(&self) -> Result<Self::Impacts<'_>> {
Ok(ImpactsImpl::new(
self.impacts.clone(),
self.doc_id_upto.clone(),
))
}
}
impl ImpactsEnum for DummyImpactsEnum {}
struct ImpactsImpl {
impacts: Rc<Vec<Vec<Impact>>>,
doc_id_upto: Rc<Vec<i32>>,
}
impl ImpactsImpl {
fn new(impacts: Rc<Vec<Vec<Impact>>>, doc_id_upto: Rc<Vec<i32>>) -> Self {
Self {
impacts,
doc_id_upto,
}
}
}
impl Impacts for ImpactsImpl {
fn num_levels(&self) -> usize {
self.impacts.len()
}
fn get_doc_id_upto(&self, level: usize) -> i32 {
self.doc_id_upto[level]
}
fn get_impacts(&self, level: usize) -> Result<Vec<Impact>> {
Ok(self.impacts[level].clone())
}
}
#[test]
fn test_random_top_docs() -> Result<()> {
let mut random = random();
let dir = new_directory_shared(&mut random)?;
let config = new_index_writer_config(&mut random)?;
let writer = RandomIndexWriter::with_config(&mut random, dir.clone(), config);
let num_docs = if is_night_mode() {
at_least(&mut random, 128 * 8 * 8 * 3)
} else {
at_least(&mut random, 100)
};
for _ in 0..num_docs {
let mut doc = Document::new();
let shift = random.random_range(0..5);
let num_terms = random.random_range(0..(1 << shift));
let mut text = String::new();
for i in 0..num_terms {
if i > 0 {
text.push(' ');
}
if random.random_range(0..2) == 0 {
text.push('a');
} else if random.random_range(0..2) == 0 {
text.push('b');
} else {
text.push('c');
}
}
doc.add(TextField::from_string("foo", text, Store::No)?);
writer.add_document(&mut random, doc)?;
}
let reader = writer.get_reader(&mut random)?;
writer.close(&mut random)?;
let searcher = new_searcher_with_reader(reader)?;
for first_term in &["a", "b", "c"] {
for second_term in &["a", "b", "c"] {
let query: Query = PhraseQuery::from_terms(0, "foo", [first_term, second_term])?.into();
let complete_manager = TopScoreDocCollectorManager::new(10, i32::MAX as usize)?;
let top_scores_manager = TopScoreDocCollectorManager::new(10, 10)?;
let complete = searcher.search_with_collector_manager(query.clone(), &complete_manager)?;
let top_scores =
searcher.search_with_collector_manager(query.clone(), &top_scores_manager)?;
CheckHits::check_equal(&query, complete.score_docs(), top_scores.score_docs())?;
let mut filtered_builder = Builder::new();
filtered_builder.add(query.clone(), Occur::Must)?;
filtered_builder.add(TermQuery::new(Term::from_text("foo", "b")), Occur::Filter)?;
let filtered_query: Query = filtered_builder.build().into();
let complete_manager = TopScoreDocCollectorManager::new(10, i32::MAX as usize)?;
let top_scores_manager = TopScoreDocCollectorManager::new(10, 10)?;
let complete =
searcher.search_with_collector_manager(filtered_query.clone(), &complete_manager)?;
let top_scores =
searcher.search_with_collector_manager(filtered_query, &top_scores_manager)?;
CheckHits::check_equal(&query, complete.score_docs(), top_scores.score_docs())?;
}
}
Ok(())
}
#[test]
#[ignore = "Java-only: Rust Term values and references cannot be null"]
fn test_null_term() -> Result<()> {
test_not_required_in_rust_lucene!();
}