rlucene 0.1.0-alpha.3

A Rust port of Apache Lucene, a full-text search engine library
Documentation
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
use crate::core::index::BytesRef;
use crate::core::index::index_reader::IndexReader;
use crate::core::index::index_reader_context::IndexReaderContext;
use crate::core::index::leaf_reader::LeafReader;
use crate::core::index::leaf_reader_context::LeafReaderContext;
use crate::core::index::term::Term;
use crate::core::index::term_states::TermStates;
use crate::core::index::terms_enum::TermsEnum;
use crate::core::search::boolean_clause::Occur;
use crate::core::search::boost_attribute::DEFAULT_BOOST;
use crate::core::search::boost_query::BoostQuery;
use crate::core::search::constant_score_query::ConstantScoreQuery;
use crate::core::search::index_searcher::IndexSearcher;
use crate::core::search::multi_term_query::{MultiTermQuery, MultiTermQuerySet, RewriteMethod};
use crate::core::search::query::Query;
use crate::core::search::similarities_impl::similarities::Similarity;
use crate::core::search::term_collecting_rewrite::{TermCollectingRewrite, TermCollector};
use crate::core::search::term_query::TermQuery;
use crate::core::search::{boolean_query, index_searcher};
use crate::core::util::allocator_byte::DirectAllocatorByte;
use crate::core::util::array_util::ArrayUtil;
use crate::core::util::attribute_source::AttributeSource;
use crate::core::util::bytes_ref_hash::{BytesRefHash, BytesStartArray, DirectBytesStartArray};
use crate::core::util::error::lucene_error::{LuceneErrorKind, Result};
use crate::core::util::{ByteBlockPool, SharedCounter};
use std::sync::Arc;
/// Base rewrite method that translates each term into a query, and keeps the scores as computed by
/// the query.
pub trait ScoringRewrite: TermCollectingRewrite {
  fn default_rewrite<IRC, Q, SIM>(
    &self,
    index_searcher: &IndexSearcher<IRC, SIM>,
    query: &Q,
  ) -> Result<Query>
  where
    SIM: Similarity,
    Q: MultiTermQuery + Into<MultiTermQuerySet>,
    IRC: IndexReaderContext,
    Self: Sized,
    IndexSearcher<IRC, SIM>: Sync,
  {
    let reader = index_searcher.get_index_reader();
    let mut builder = self.get_top_level_builder()?;

    let mut col = ParallelArraysTermCollector::new(|size| self.check_max_clause_count(size))?;

    self.collect_terms(index_searcher.get_top_reader_context(), query, &mut col)?;

    let size = col.terms.size();
    #[allow(clippy::needless_range_loop)]
    if size > 0 {
      col.terms.sort(&col.block_pool)?;
      let sort = col.terms.ids.as_slice();
      for i in 0..size {
        let pos = sort[i];
        let position = col.terms.get(pos, &col.block_pool)?;
        let block = col.block_pool.get_buffer(position.block_index);
        // The query clause retains this term after the collector is dropped.
        let term = Term::new(
          query.get_field(),
          &block[position.offset..position.offset + position.length],
        );
        let pos = pos as usize;
        let term_state = std::mem::take(&mut col.terms.bytes_start_array.term_state[pos]);

        debug_assert_eq!(reader.doc_freq(&term)?, term_state.doc_freq()?);
        let doc_freq = term_state.doc_freq()?;
        self.add_clause_with_states(
          &mut builder,
          term,
          doc_freq,
          col.terms.bytes_start_array.boost[pos],
          Some(term_state),
        )?;
      }
    }

    self.build(builder)
  }
  /// This method is called after every new term to check if the number of max clauses (e.g. in
  /// [`BooleanQuery`]) is not exceeded. Returns the corresponding error.
  fn check_max_clause_count(&self, count: usize) -> Result<()>;
}
/// A rewrite method that first translates each term into [`Occur::Should`] clause
/// in a [`BooleanQuery`], and keeps the scores as computed by the query. Note that typically such
/// scores are meaningless to the user, and require non-trivial CPU to compute, so it's almost
/// always better to use [`MultiTermQuery::CONSTANT_SCORE_BLENDED_REWRITE`] or
/// [`MultiTermQuery::CONSTANT_SCORE_REWRITE`] instead.
///
/// **NOTE**: This rewrite method will return
/// [`LuceneErrorKind::TooManyClauses`](crate::core::util::error::lucene_error::LuceneErrorKind::TooManyClauses)
/// if the number of terms exceeds [`index_searcher::get_max_clause_count`](crate::core::search::index_searcher::get_max_clause_count).
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct ScoringBooleanRewrite;
impl TermCollectingRewrite for ScoringBooleanRewrite {
  type B = boolean_query::Builder;

  fn get_top_level_builder(&self) -> Result<Self::B> {
    Ok(boolean_query::Builder::new())
  }

  fn build(&self, builder: Self::B) -> Result<Query> {
    Ok(builder.build().into())
  }

  fn add_clause_with_states(
    &self,
    top_level: &mut Self::B,
    term: Term,
    _doc_count: i32,
    boost: f32,
    states: Option<TermStates>,
  ) -> Result<()> {
    let tq = TermQuery::with_term_state(term, states.map(Arc::new));
    top_level.add(BoostQuery::new(tq, boost)?, Occur::Should)?;
    Ok(())
  }
}

impl RewriteMethod for ScoringBooleanRewrite {
  fn rewrite<IRC, Q, SIM>(
    &self,
    index_searcher: &IndexSearcher<IRC, SIM>,
    query: &Q,
  ) -> Result<Query>
  where
    SIM: Similarity,
    Q: MultiTermQuery + Into<MultiTermQuerySet>,
    IRC: IndexReaderContext,
    IndexSearcher<IRC, SIM>: Sync,
  {
    self.default_rewrite(index_searcher, query)
  }
}

impl ScoringRewrite for ScoringBooleanRewrite {
  fn check_max_clause_count(&self, count: usize) -> Result<()> {
    if count > index_searcher::get_max_clause_count() {
      return Err(index_searcher::new());
    }
    Ok(())
  }
}
/// Like [`SCORING_BOOLEAN_REWRITE`](crate::core::search::multi_term_query::SCORING_BOOLEAN_REWRITE) except scores are not computed. Instead, each matching
/// document receives a constant score equal to the query's boost.
///
/// **NOTE**: This rewrite method will return
/// [`LuceneErrorKind::TooManyClauses`](crate::core::util::error::lucene_error::LuceneErrorKind::TooManyClauses)
/// if the number of terms exceeds [`index_searcher::get_max_clause_count`](crate::core::search::index_searcher::get_max_clause_count).
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct ConstantScoreBooleanRewrite;
impl RewriteMethod for ConstantScoreBooleanRewrite {
  fn rewrite<IRC, Q, SIM>(
    &self,
    index_searcher: &IndexSearcher<IRC, SIM>,
    query: &Q,
  ) -> Result<Query>
  where
    SIM: Similarity,
    Q: MultiTermQuery + Into<MultiTermQuerySet>,
    IRC: IndexReaderContext,
    IndexSearcher<IRC, SIM>: Sync,
  {
    let bq = ScoringBooleanRewrite.rewrite(index_searcher, query)?;
    Ok(ConstantScoreQuery::new(bq).into())
  }
}

struct ParallelArraysTermCollector<F> {
  terms: BytesRefHash<TermFreqBoostByteStart>,
  block_pool: ByteBlockPool,
  ord: usize,
  check_max_clause_count: F,
}
impl<F> ParallelArraysTermCollector<F> {
  fn new(check_max_clause_count: F) -> Result<Self> {
    let allocator = DirectAllocatorByte::new();
    let block_pool = ByteBlockPool::new(allocator);
    let array = TermFreqBoostByteStart::new(16);
    let terms = BytesRefHash::from_bytes_start_array(16, array)?;
    Ok(Self {
      terms,
      block_pool,
      ord: 0,
      check_max_clause_count,
    })
  }
}
impl<F> TermCollector for ParallelArraysTermCollector<F>
where
  F: FnMut(usize) -> Result<()>,
{
  fn set_reader_context<LR>(&mut self, context: &LeafReaderContext<LR>) -> Result<()>
  where
    LR: LeafReader,
  {
    self.ord = context.ord;
    Ok(())
  }

  fn collect<TE, IRC>(
    &mut self,
    bytes: &BytesRef<Vec<u8>>,
    terms_enum: &mut TE,
    top_reader_context: &IRC,
  ) -> Result<bool>
  where
    TE: TermsEnum,
    IRC: IndexReaderContext,
  {
    let e = self.terms.add(bytes, &mut self.block_pool)?;
    let state = terms_enum.term_state()?;

    if e < 0 {
      let pos = (-e - 1) as usize;

      let array = &mut self.terms.bytes_start_array;

      array.term_state[pos].register_with_stats(
        state,
        self.ord,
        terms_enum.doc_freq()?,
        terms_enum.total_term_freq()?,
      );
      debug_assert_eq!(
        array.boost[pos],
        match (|| -> Result<f32> {
          let attr = terms_enum.attributes()?;
          attr.get_boost()
        })() {
          Ok(boost) => boost,
          Err(error) if matches!(error.kind(), LuceneErrorKind::UnsupportedOperation(_)) =>
            DEFAULT_BOOST,
          Err(e) => return Err(e),
        },
        "boost should be equal in all segment TermsEnums"
      );
    } else {
      let pos = e as usize;
      let array = &mut self.terms.bytes_start_array;
      let boost = match (|| -> Result<f32> {
        let attr = terms_enum.attributes()?;
        attr.get_boost()
      })() {
        Ok(boost) => boost,
        Err(error) if matches!(error.kind(), LuceneErrorKind::UnsupportedOperation(_)) => {
          DEFAULT_BOOST
        },
        Err(e) => return Err(e),
      };
      debug_assert_eq!(pos, array.boost.len());
      array.boost.push(boost);
      array.term_state.push(TermStates::with_state_and_stats(
        top_reader_context,
        state,
        self.ord,
        terms_enum.doc_freq()?,
        terms_enum.total_term_freq()?,
      )?);
      (self.check_max_clause_count)(self.terms.size())?;
    }

    Ok(true)
  }

  fn set_next_enum<TE>(&mut self, _terms_enum: &mut TE) -> Result<()>
  where
    TE: TermsEnum,
  {
    Ok(())
  }
}
/// Special implementation of BytesStartArray that keeps parallel arrays for boost and docFreq
struct TermFreqBoostByteStart {
  boost: Vec<f32>,
  // New term IDs are consecutive; only successfully collected states occupy the prefix.
  term_state: Vec<TermStates>,
  base: DirectBytesStartArray,
}
impl TermFreqBoostByteStart {
  fn new(init_size: usize) -> Self {
    let base = DirectBytesStartArray::new(init_size);
    Self {
      boost: Vec::new(),
      term_state: Vec::new(),
      base,
    }
  }
}
impl BytesStartArray for TermFreqBoostByteStart {
  fn init(&mut self) -> Result<()> {
    self.base.init()?;
    let len = self.base.bytes_start.as_slice().len();
    ArrayUtil::grow_capacity(&mut self.boost, len)?;
    if self.term_state.capacity() < len {
      let capacity = ArrayUtil::oversize(len, size_of::<TermStates>())?;
      self.term_state.reserve(capacity - self.term_state.len());
    }
    debug_assert!(self.term_state.capacity() >= len);
    debug_assert!(self.boost.capacity() >= len);
    Ok(())
  }

  fn grow(&mut self) -> Result<()> {
    self.base.grow()?;
    let ord = self.base.bytes_start.as_slice();
    let len = ord.len();
    ArrayUtil::grow_capacity(&mut self.boost, len)?;
    if self.term_state.capacity() < len {
      self.term_state.reserve(len - self.term_state.len());
    }
    Ok(())
  }

  fn clear(&mut self) {
    self.boost.clear();
    self.term_state.clear();
    self.base.clear();
  }

  fn bytes_used(&mut self) -> SharedCounter {
    self.base.bytes_used()
  }

  fn get_value(&self, index: usize) -> Result<i32> {
    self.base.get_value(index)
  }

  fn set_value(&mut self, index: usize, value: i32) -> Result<()> {
    self.base.set_value(index, value)
  }

  fn len(&self) -> Result<usize> {
    self.base.len()
  }

  fn need_init(&self) -> bool {
    self.base.need_init()
  }

  fn ram_bytes_used(&self) -> Result<i64> {
    self.base.ram_bytes_used()
  }
}