rlucene 0.1.0-alpha.3

A Rust port of Apache Lucene, a full-text search engine library
Documentation
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
use crate::core::document::document::Document;
use crate::core::document::field::Store;
use crate::core::document::lat_lon_doc_values_field::LatLonDocValuesField;
use crate::core::document::lat_lon_point::LatLonPoint;
use crate::core::document::stored_field::StoredField;
use crate::core::document::string_field::StringField;
use crate::core::geo::geo_encoding_utils::GeoEncodingUtils;
use crate::core::index::directory_reader;
use crate::core::index::index_reader::IndexReader;
use crate::core::index::index_reader_context::IndexReaderContext;
use crate::core::index::index_writer::IndexWriter;
use crate::core::index::index_writer_config::IndexWriterConfig;
use crate::core::index::indexable_field::IndexableField;
use crate::core::index::live_index_writer_config::LiveIndexWriterConfig;
use crate::core::index::serial_merge_scheduler::SerialMergeScheduler;
use crate::core::index::stored_fields::StoredFields;
use crate::core::index::term::Term;
use crate::core::search::match_all_docs_query::MatchAllDocsQuery;
use crate::core::search::score_doc::ScoreDocLike;
use crate::core::search::sort::Sort;
use crate::core::search::top_docs::TopDocsLike;
use crate::core::store::directory::Directory;
use crate::core::util::error::lucene_error::Result;
use crate::core::util::{CoreHelper, SloppyMath};
use crate::test_framework::core::geo::geo_test_util::GeoTestUtil;
use crate::test_framework::core::index::random_index_writer::RandomIndexWriter;
use crate::test_framework::core::util::lucene_test_case::{
  at_least, at_least_usize, create_temp_dir_with_prefix, new_directory_shared, new_fs_directory,
  new_index_writer_config, new_log_merge_policy, new_searcher_with_reader, random,
};
use crate::test_framework::core::util::test_util::TestUtil;
use rand::{Rng, RngExt};
use std::cmp::Ordering;
#[allow(dead_code)] // for quick search
struct TestNearest;
#[test]
fn test_nearest_neighbor_with_deleted_docs() -> Result<()> {
  let mut random = random();
  let dir = new_directory_shared(&mut random)?;
  let iwc = get_index_writer_config(&mut random)?;
  let w = RandomIndexWriter::with_config(&mut random, dir.clone(), iwc);

  let mut doc = Document::new();
  doc.add(LatLonPoint::new("point", 40.0, 50.0)?);
  doc.add(StringField::from_string("id", "0", Store::Yes)?);
  w.add_document(&mut random, doc)?;

  let mut doc = Document::new();
  doc.add(LatLonPoint::new("point", 45.0, 55.0)?);
  doc.add(StringField::from_string("id", "1", Store::Yes)?);
  w.add_document(&mut random, doc)?;

  let r = w.get_reader(&mut random)?;
  // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps
  // with its own points impl:
  let mut s = new_searcher_with_reader(r)?;
  let top_field_docs = LatLonPoint::nearest(&s, "point", 40.0, 50.0, 1)?;
  let hit = top_field_docs.score_docs()[0].as_field().unwrap();
  assert_eq!(
    "0",
    s.reader_context
      .reader()
      .stored_fields()?
      .document(hit.doc())?
      .get_field("id")
      .unwrap()
      .string_value()?
      .unwrap()
      .as_ref()
  );
  s.reader_context.reader().close()?;

  w.delete_documents_with_terms(&mut random, vec![Term::from_text("id", "0")])?;
  let r = w.get_reader(&mut random)?;
  // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps
  // with its own points impl:
  s = new_searcher_with_reader(r)?;
  let top_field_docs = LatLonPoint::nearest(&s, "point", 40.0, 50.0, 1)?;
  let hit_ref = top_field_docs.score_docs()[0].as_field();
  let hit = hit_ref.as_ref().unwrap();
  assert_eq!(
    "1",
    s.reader_context
      .reader()
      .stored_fields()?
      .document(hit.doc())?
      .get_field("id")
      .unwrap()
      .string_value()?
      .unwrap()
      .as_ref()
  );
  s.reader_context.reader().close()?;
  w.close(&mut random)?;
  Ok(())
}
#[test]
fn test_nearest_neighbor_with_all_deleted_docs() -> Result<()> {
  let mut random = random();
  let dir = new_directory_shared(&mut random)?;
  let iwc = get_index_writer_config(&mut random)?;
  let w = RandomIndexWriter::with_config(&mut random, dir.clone(), iwc);

  let mut doc = Document::new();
  doc.add(LatLonPoint::new("point", 40.0, 50.0)?);
  doc.add(StringField::from_string("id", "0", Store::Yes)?);
  w.add_document(&mut random, doc)?;

  let mut doc = Document::new();
  doc.add(LatLonPoint::new("point", 45.0, 55.0)?);
  doc.add(StringField::from_string("id", "1", Store::Yes)?);
  w.add_document(&mut random, doc)?;

  let r = w.get_reader(&mut random)?;
  // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps
  // with its own points impl:
  let mut s = new_searcher_with_reader(r)?;
  let top_field_docs = LatLonPoint::nearest(&s, "point", 40.0, 50.0, 1)?;
  let hit = top_field_docs.score_docs()[0].as_field().unwrap();
  assert_eq!(
    "0",
    s.reader_context
      .reader()
      .stored_fields()?
      .document(hit.doc())?
      .get_field("id")
      .unwrap()
      .string_value()?
      .unwrap()
      .as_ref()
  );
  s.reader_context.reader().close()?;

  w.delete_documents_with_terms(&mut random, vec![Term::from_text("id", "0")])?;
  w.delete_documents_with_terms(&mut random, vec![Term::from_text("id", "1")])?;

  let r = w.get_reader(&mut random)?;
  // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps
  // with its own points impl:
  s = new_searcher_with_reader(r)?;
  assert_eq!(
    0,
    LatLonPoint::nearest(&s, "point", 40.0, 50.0, 1)?
      .score_docs()
      .len()
  );
  s.reader_context.reader().close()?;
  w.close(&mut random)?;
  Ok(())
}

#[test]
fn test_tie_break_by_doc_id() -> Result<()> {
  let mut random = random();
  let dir = new_directory_shared(&mut random)?;
  let iwc = get_index_writer_config(&mut random)?;
  let w = IndexWriter::new(dir.clone(), iwc)?;

  let mut doc = Document::new();
  doc.add(LatLonPoint::new("point", 40.0, 50.0)?);
  doc.add(StringField::from_string("id", "0", Store::Yes)?);
  w.add_document(doc)?;

  let mut doc = Document::new();
  doc.add(LatLonPoint::new("point", 40.0, 50.0)?);
  doc.add(StringField::from_string("id", "1", Store::Yes)?);
  w.add_document(doc)?;

  let r = directory_reader::open_from_writer(&w)?;
  // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps
  // with its own points impl:
  let searcher = new_searcher_with_reader(r)?;
  let top_field_docs = LatLonPoint::nearest(&searcher, "point", 45.0, 50.0, 2)?;

  let hit = top_field_docs.score_docs()[0].as_field().unwrap();
  assert_eq!(
    "0",
    searcher
      .reader_context
      .reader()
      .stored_fields()?
      .document(hit.doc())?
      .get_field("id")
      .unwrap()
      .string_value()?
      .unwrap()
      .as_ref()
  );

  let hit = top_field_docs.score_docs()[1].as_field().unwrap();
  assert_eq!(
    "1",
    searcher
      .reader_context
      .reader()
      .stored_fields()?
      .document(hit.doc())?
      .get_field("id")
      .unwrap()
      .string_value()?
      .unwrap()
      .as_ref()
  );

  searcher.reader_context.reader().close()?;
  w.close()?;
  Ok(())
}

#[test]
fn test_nearest_neighbor_with_no_docs() -> Result<()> {
  let mut random = random();
  let dir = new_directory_shared(&mut random)?;
  let iwc = get_index_writer_config(&mut random)?;
  let w = RandomIndexWriter::with_config(&mut random, dir.clone(), iwc);

  let r = w.get_reader(&mut random)?;
  // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps
  // with its own points impl:
  let searcher = new_searcher_with_reader(r)?;
  assert_eq!(
    0,
    LatLonPoint::nearest(&searcher, "point", 40.0, 50.0, 1)?
      .score_docs()
      .len()
  );
  searcher.reader_context.reader().close()?;
  w.close(&mut random)?;
  Ok(())
}

fn quantize_lat(lat_raw: f64) -> Result<f64> {
  Ok(GeoEncodingUtils::decode_latitude(
    GeoEncodingUtils::encode_latitude(lat_raw)?,
  ))
}

fn quantize_lon(lon_raw: f64) -> Result<f64> {
  Ok(GeoEncodingUtils::decode_longitude(
    GeoEncodingUtils::encode_longitude(lon_raw)?,
  ))
}

#[test]
fn test_nearest_neighbor_random() -> Result<()> {
  let mut random = random();

  let num_points = at_least_usize(&mut random, 1000);
  let dir = if num_points > 100_000 {
    new_fs_directory(&mut random, create_temp_dir_with_prefix("TestNearest")?)?
  } else {
    new_directory_shared(&mut random)?
  };

  let mut lats = vec![0.0; num_points];
  let mut lons = vec![0.0; num_points];

  let mut iwc = get_index_writer_config(&mut random)?;
  iwc.set_merge_policy(new_log_merge_policy(&mut random)?);
  iwc.set_merge_scheduler(SerialMergeScheduler::new());
  let w = RandomIndexWriter::with_config(&mut random, dir.clone(), iwc);

  for id in 0..num_points {
    lats[id] = quantize_lat(GeoTestUtil::next_latitude(&mut random))?;
    lons[id] = quantize_lon(GeoTestUtil::next_longitude(&mut random))?;

    let mut doc = Document::new();
    doc.add(LatLonPoint::new("point", lats[id], lons[id])?);
    doc.add(LatLonDocValuesField::new("point", lats[id], lons[id])?);
    doc.add(StoredField::from_i32("id", id as i32)?);
    w.add_document(&mut random, doc)?;
  }

  if random.random_bool(0.5) {
    w.force_merge(&mut random, 1)?;
  }

  let r = w.get_reader(&mut random)?;

  // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps
  // with its own points impl:
  let s = new_searcher_with_reader(r)?;
  let iters = at_least(&mut random, 100);

  for iter in 0..iters {
    if cfg!(feature = "test_log_verbose") {
      println!("\nTEST: iter={}", iter);
    }

    let point_lat = GeoTestUtil::next_latitude(&mut random);
    let point_lon = GeoTestUtil::next_longitude(&mut random);

    // dumb brute force search to get the expected result:
    let mut expected_hits = Vec::with_capacity(lats.len());
    for id in 0..lats.len() {
      let distance = SloppyMath::haversin_meters(point_lat, point_lon, lats[id], lons[id]);
      expected_hits.push((id as i32, distance));
    }

    expected_hits.sort_by(|a, b| {
      let cmp = CoreHelper::compare_f64(a.1, b.1);
      if cmp != Ordering::Equal {
        return cmp;
      }

      // tie break by smaller docID:
      a.0.cmp(&b.0)
    });

    let top_n = TestUtil::next_usize(&mut random, 1, lats.len());
    // Also test with MatchAllDocsQuery, sorting by distance:
    let field_docs = s.search_with_sort(
      MatchAllDocsQuery::new(),
      top_n,
      Sort::with_fields(vec![LatLonDocValuesField::new_distance_sort(
        "point", point_lat, point_lon,
      )?])?,
    )?;

    let hits = LatLonPoint::nearest(&s, "point", point_lat, point_lon, top_n as i32)?;
    let mut stored_fields = s.reader_context.reader().stored_fields()?;

    #[allow(clippy::needless_range_loop)]
    for i in 0..top_n {
      let expected = &expected_hits[i];
      let expected2 = field_docs.score_docs()[i].as_field().unwrap();
      let actual = hits.score_docs()[i].as_field().unwrap();
      let _actual_doc = stored_fields.document(actual.doc())?;

      assert_eq!(expected.0, actual.doc());
      assert_eq!(expected.1, *actual.fields[0].as_f64().unwrap());

      assert_eq!(expected2.doc(), actual.doc());
      assert_eq!(
        *expected2.fields[0].as_f64().unwrap(),
        *actual.fields[0].as_f64().unwrap()
      );
    }
  }

  s.reader_context.reader().close()?;
  w.close(&mut random)?;
  Ok(())
}
fn get_index_writer_config<D, R>(random: &mut R) -> Result<IndexWriterConfig<D>>
where
  D: Directory,
  R: Rng + ?Sized,
{
  let mut iwc = new_index_writer_config(random)?;
  iwc.set_codec(TestUtil::get_default_codec());
  Ok(iwc)
}