use crate::core::document::document::Document;
use crate::core::document::field::Store;
use crate::core::document::text_field::TextField;
use crate::core::index::directory_reader;
use crate::core::index::field_invert_state::FieldInvertState;
use crate::core::index::index_reader::IndexReader;
use crate::core::index::index_writer::{IndexWriter, MAX_TERM_LENGTH};
use crate::core::index::leaf_reader::LeafReader;
use crate::core::index::live_index_writer_config::LiveIndexWriterConfig;
use crate::core::index::multi_doc_values::MultiDocValues;
use crate::core::index::numeric_doc_values::NumericDocValues;
use crate::core::index::stored_fields::StoredFields;
use crate::core::search::collection_statistics::CollectionStatistics;
use crate::core::search::doc_id_set_iterator::DocIdSetIterator;
use crate::core::search::similarities_impl::classic_similarity;
use crate::core::search::similarities_impl::per_field_similarity_wrapper::PerFieldSimilarityWrapper;
use crate::core::search::similarities_impl::similarities::{BoxSimScorer, Similarity};
use crate::core::search::similarities_impl::tf_idf_similarity::TFIDFSimilarity;
use crate::core::search::term_statistics::TermStatistics;
use crate::core::store::directory::DirEnum;
use crate::core::util::close::CloseableRef;
use crate::core::util::error::lucene_error::{LuceneError, Result};
use crate::test_framework::core::analysis::mock_analyzer::MockAnalyzer;
use crate::test_framework::core::index::random_index_writer::RandomIndexWriter;
use crate::test_framework::core::util::lucene_test_case::{
at_least, create_temp_dir_with_prefix, get_only_leaf_reader, new_directory_shared,
new_fs_directory, new_index_writer_config, new_index_writer_config_with_analyzer,
new_log_merge_policy, new_text_field, random,
};
use crate::test_framework::core::util::test_util::TestUtil;
use rand::rngs::StdRng;
use std::collections::HashMap;
use std::fmt::{Display, Formatter};
use std::sync::Arc;
#[allow(dead_code)] pub struct TestNorms;
const BYTE_TEST_FIELD: &str = "byte_norms";
#[test]
fn test_max_byte_norms() -> Result<()> {
let mut random = random();
let dir = new_fs_directory(
&mut random,
create_temp_dir_with_prefix("TestNorms.testMaxByteNorms")?,
)?;
build_index(dir.clone(), &mut random)?;
let open = directory_reader::open(dir.clone())?;
let mut norm_values = MultiDocValues::get_norm_values(&open, BYTE_TEST_FIELD)?
.ok_or_else(|| LuceneError::illegal_state("norm_values is None"))?;
let mut stored_fields = open.stored_fields()?;
for i in 0..open.max_doc()? {
let document = stored_fields.document(i)?;
let field_value = document
.get(BYTE_TEST_FIELD)?
.ok_or_else(|| LuceneError::illegal_state("field value is None"))?;
let expected: i64 = field_value
.split(' ')
.next()
.ok_or_else(|| LuceneError::illegal_state("missing first token"))?
.parse()?;
assert_eq!(i, norm_values.next_doc()?);
assert_eq!(expected, norm_values.long_value()?);
}
open.close()?;
dir.close()?;
Ok(())
}
pub fn build_index(dir: Arc<DirEnum>, random: &mut StdRng) -> Result<()> {
let mut analyzer = MockAnalyzer::new(random);
analyzer.set_max_token_length(TestUtil::next_int(random, 3, MAX_TERM_LENGTH));
let provider = MySimProvider;
let indexing_similarity_config = provider;
let config =
crate::test_framework::core::util::lucene_test_case::new_index_writer_config_with_analyzer(
random, analyzer,
)?;
let writer = RandomIndexWriter::with_config(random, dir, config);
let num = at_least(random, 100);
for _ in 0..num {
let mut doc = Document::new();
let boost = TestUtil::next_int(random, 1, 255);
let boost_text = boost.to_string();
let value = (0..boost)
.map(|_| boost_text.as_str())
.collect::<Vec<_>>()
.join(" ");
let field = TextField::from_string(BYTE_TEST_FIELD, value, Store::Yes)?;
doc.add(field);
writer.add_document_with_similarity(random, doc, &indexing_similarity_config)?;
}
writer.commit(random)?;
writer.close(random)?;
Ok(())
}
#[test]
fn test_empty_value_vs_no_value() -> Result<()> {
let mut random = random();
let dir = new_directory_shared(&mut random)?;
let mut iwc = new_index_writer_config(&mut random)?;
iwc.set_merge_policy(new_log_merge_policy(&mut random)?);
let writer = IndexWriter::new(dir.clone(), iwc)?;
let mut doc = Document::new();
writer.add_document(&mut doc)?;
let mut field_to_type = HashMap::new();
doc.add(new_text_field(
&mut random,
"foo",
"",
Store::No,
&mut field_to_type,
)?);
writer.add_document(&mut doc)?;
writer.force_merge(1)?;
let reader = directory_reader::open_from_writer(&writer)?;
writer.close()?;
let leaf_reader = get_only_leaf_reader(&reader)?;
let mut norm_values = leaf_reader.get_norm_values("foo")?.unwrap();
assert_eq!(1, norm_values.next_doc()?); assert_eq!(0, norm_values.long_value()?);
reader.close()?;
dir.close()?;
Ok(())
}
#[derive(Default, Clone)]
pub struct ByteEncodingBoostSimilarity;
impl Display for ByteEncodingBoostSimilarity {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", std::any::type_name::<Self>())
}
}
impl Similarity for ByteEncodingBoostSimilarity {
fn compute_norm(&self, state: &FieldInvertState) -> Result<i64> {
Ok(state.get_length() as i64)
}
type SimScorer = BoxSimScorer;
fn scorer(
&self,
_boost: f32,
_collection_stats: &CollectionStatistics,
_term_stats: &[TermStatistics],
) -> Result<Self::SimScorer> {
Err(LuceneError::unsupported_operation(""))
}
}
#[derive(Default, Clone)]
struct MySimProvider;
impl Similarity for MySimProvider {
fn compute_norm(&self, state: &FieldInvertState) -> Result<i64> {
PerFieldSimilarityWrapper::compute_norm(self, state)
}
type SimScorer = BoxSimScorer;
fn scorer(
&self,
boost: f32,
collection_stats: &CollectionStatistics,
term_stats: &[TermStatistics],
) -> Result<Self::SimScorer> {
Ok(Box::new(PerFieldSimilarityWrapper::scorer(
self,
boost,
collection_stats,
term_stats,
)?))
}
}
impl Display for MySimProvider {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", std::any::type_name::<Self>())
}
}
impl PerFieldSimilarityWrapper for MySimProvider {
type Similarity = TestSimilarityEnum;
fn get(&self, field: &str) -> Self::Similarity {
if BYTE_TEST_FIELD == field {
TestSimilarityEnum::ByteEncodingBoost(ByteEncodingBoostSimilarity)
} else {
TestSimilarityEnum::Classic(classic_similarity::new())
}
}
}
pub enum TestSimilarityEnum {
ByteEncodingBoost(ByteEncodingBoostSimilarity),
Classic(TFIDFSimilarity),
}
impl Display for TestSimilarityEnum {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
TestSimilarityEnum::ByteEncodingBoost(v) => write!(f, "{}", v),
TestSimilarityEnum::Classic(v) => write!(f, "{}", v),
}
}
}
impl Similarity for TestSimilarityEnum {
fn get_discount_overlaps(&self) -> bool {
match self {
TestSimilarityEnum::ByteEncodingBoost(_) => true,
TestSimilarityEnum::Classic(_) => false,
}
}
fn compute_norm(&self, state: &FieldInvertState) -> Result<i64> {
match self {
TestSimilarityEnum::ByteEncodingBoost(v) => v.compute_norm(state),
TestSimilarityEnum::Classic(v) => v.compute_norm(state),
}
}
type SimScorer = BoxSimScorer;
fn scorer(
&self,
boost: f32,
collection_stats: &CollectionStatistics,
term_stats: &[TermStatistics],
) -> Result<Self::SimScorer> {
match self {
TestSimilarityEnum::ByteEncodingBoost(v) => v.scorer(boost, collection_stats, term_stats),
TestSimilarityEnum::Classic(v) => {
let v = v.scorer(boost, collection_stats, term_stats)?;
Ok(Box::new(v))
},
}
}
}