rlucene 0.1.0-alpha.3

A Rust port of Apache Lucene, a full-text search engine library
Documentation
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
use crate::core::index::point_values::PointValues;
use crate::core::index::terms::Terms;
use crate::core::search::doc_id_set::DocIdSet;
use crate::core::search::doc_id_set_iterator::NO_MORE_DOCS;
use crate::core::search::doc_id_set_iterator::{BitSetIteratorAccess, DocIdSetIterator};
use crate::core::util::TryIntoInt;
use crate::core::util::accountable::Accountable;
use crate::core::util::bit_doc_id_set::BitDocIdSet;
use crate::core::util::bit_set::BitSet;
use crate::core::util::bit_set_iterator::BitSetIterator;
use crate::core::util::error::lucene_error::Result;
use crate::core::util::fixed_bit_set::FixedBitSet;
use crate::core::util::int_array_doc_id_set::{IntArrayDocIdSet, IntArrayDocIdSetIterator};
use std::borrow::Borrow;
use std::rc::Rc;
use std::sync::Arc;

/// A builder of [`DocIdSet`]s. Initially, it uses a sparse structure to gather
/// documents, and then upgrades to a non-sparse bit set once enough hits match.
///
///
/// # Note
/// This is an internal API.
pub struct DocIdSetBuilder {
  max_doc: i32,
  threshold: usize,
  // Crate-visible for testing.
  pub(crate) multi_valued: bool,
  pub(crate) num_values_per_doc: f64,

  buffer: Vec<i32>,
  bit_set: Option<FixedBitSet>,
  counter: i64,
}
impl DocIdSetBuilder {
  /// Create a builder that can contain doc IDs between  0 and maxDoc.
  pub fn new(max_doc: i32) -> DocIdSetBuilder {
    Self::with_count(max_doc, -1, -1)
  }
  pub fn from_terms<T>(max_doc: i32, terms: &T) -> Result<DocIdSetBuilder>
  where
    T: Terms,
  {
    Ok(Self::with_count(
      max_doc,
      terms.get_doc_count()?,
      terms.get_sum_doc_freq()?,
    ))
  }

  pub fn from_point_values<PV>(max_doc: i32, values: &PV, _field: &str) -> Result<DocIdSetBuilder>
  where
    PV: PointValues,
  {
    let v: i64 = values.size()?.try_convert()?;
    Ok(Self::with_count(max_doc, values.get_doc_count()?, v))
  }

  pub fn with_count(max_doc: i32, doc_count: i32, value_count: i64) -> DocIdSetBuilder {
    let multi_valued = doc_count < 0 || doc_count as i64 != value_count;
    let num_values_per_doc = if doc_count <= 0 || value_count < 0 {
      // assume one value per doc, this means the cost will be
      // overestimated if the docs are actually multi-valued
      1f64
    } else {
      // otherwise compute from index stats
      value_count as f64 / doc_count as f64
    };
    debug_assert!(
      num_values_per_doc >= 1f64,
      "value_count = {value_count} doc_count = {doc_count}"
    );
    // For very small sets, use a sorted `Vec<i32>`.
    // maxDoc >>> 7 is a good value if you want to save memory, lower values
    // such as maxDoc >>> 11 should provide faster building but at the
    // expense of using a full bitset even for quite sparse data
    Self {
      max_doc,
      multi_valued,
      num_values_per_doc,
      threshold: (max_doc >> 7) as usize,
      buffer: Vec::new(),
      bit_set: None,
      counter: 0,
    }
  }
  pub fn add_disi<I>(&mut self, iter: &mut I) -> Result<()>
  where
    I: DocIdSetIterator,
  {
    let cost = std::cmp::min(iter.cost()?, i32::MAX as i64);
    self.grow(cost as usize)?;
    if let Some(bit_set) = self.bit_set.as_mut() {
      BitSet::or(bit_set, iter)?;
      return Ok(());
    }
    let buffer = &mut self.buffer;
    for _i in 0..cost {
      let doc = iter.next_doc()?;
      if doc == NO_MORE_DOCS {
        return Ok(());
      }
      buffer.push(doc);
    }
    let mut doc = iter.next_doc()?;
    while doc != NO_MORE_DOCS {
      self.grow(1)?;
      self.add_doc(doc)?;
      doc = iter.next_doc()?;
    }
    Ok(())
  }
  pub fn add_doc(&mut self, doc: i32) -> Result<()> {
    if let Some(bit_set) = self.bit_set.as_mut() {
      bit_set.set(doc as usize)?;
    } else {
      self.buffer.push(doc);
    }
    Ok(())
  }
  pub fn grow(&mut self, num_docs: usize) -> Result<()> {
    if self.bit_set.is_none() {
      if self.buffer.len() + num_docs > self.threshold {
        self.upgrade_to_bitset()?;
        self.counter += num_docs as i64;
      }
    } else {
      self.counter += num_docs as i64;
    }
    Ok(())
  }
  fn upgrade_to_bitset(&mut self) -> Result<()> {
    debug_assert!(self.bit_set.is_none());
    let mut bitset = FixedBitSet::new(self.max_doc as usize);
    let mut counter = 0i64;
    for doc in self.buffer.iter() {
      bitset.set(*doc as usize)?;
      counter += 1;
    }
    self.bit_set = Some(bitset);
    self.counter = counter;
    self.buffer.clear();
    Ok(())
  }
  pub fn build(&mut self) -> Result<DocIdSetBuilderEnum> {
    self.build_with(
      |bits, cost| {
        Ok(DocIdSetBuilderEnum::BitDoc(BitDocIdSet::with_cost(
          Some(bits),
          cost,
        )?))
      },
      |docs, length| {
        Ok(DocIdSetBuilderEnum::IntArray(IntArrayDocIdSet::new(
          docs, length,
        )?))
      },
    )
  }

  /// Moves the completed doc IDs into a single independent iterator.
  pub fn build_iterator(&mut self) -> Result<OwnedDocIdSetBuilderIterator> {
    self.build_with(
      |bits, cost| {
        Ok(DocIdSetBuilderIterator::BitSet(BitSetIterator::new(
          bits, cost,
        )?))
      },
      |docs, length| {
        Ok(DocIdSetBuilderIterator::IntArray(
          IntArrayDocIdSetIterator::new(docs, length),
        ))
      },
    )
  }

  fn build_with<T, FB, FA>(&mut self, bit_set: FB, int_array: FA) -> Result<T>
  where
    FB: FnOnce(FixedBitSet, i64) -> Result<T>,
    FA: FnOnce(Vec<i32>, usize) -> Result<T>,
  {
    let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
      if self.bit_set.is_some() {
        debug_assert!(self.counter >= 0);
        let cost = (self.counter as f64 / self.num_values_per_doc).round();
        let bits = self.bit_set.take().ok_or_else(|| {
          crate::core::util::error::lucene_error::LuceneError::illegal_state(
            "missing builder bit set",
          )
        })?;
        bit_set(bits, cost as i64)
      } else {
        self.buffer.sort_unstable();
        if self.multi_valued {
          self.buffer.dedup();
        } else {
          debug_assert!(self.no_dups());
        }
        self.buffer.push(NO_MORE_DOCS);
        let l = self.buffer.len() - 1;
        int_array(std::mem::take(&mut self.buffer), l)
      }
    }));
    self.buffer.clear();
    self.bit_set = None;
    unwrap_caught_result!(result)
  }
  fn no_dups(&self) -> bool {
    for i in 1..self.buffer.len() {
      debug_assert!(self.buffer[i] > self.buffer[i - 1])
    }
    true
  }
  #[cfg(test)]
  pub fn get_num_values_per_doc(&self) -> f64 {
    self.num_values_per_doc
  }
  #[cfg(test)]
  pub fn get_multi_valued(&self) -> bool {
    self.multi_valued
  }
}

pub enum DocIdSetBuilderEnum {
  BitDoc(BitDocIdSet<FixedBitSet>),
  IntArray(IntArrayDocIdSet),
}
impl Accountable for DocIdSetBuilderEnum {
  fn ram_bytes_used(&self) -> Result<i64> {
    match self {
      Self::BitDoc(set) => set.ram_bytes_used(),
      Self::IntArray(set) => set.ram_bytes_used(),
    }
  }
}

impl DocIdSet for DocIdSetBuilderEnum {
  type DocIdSetIterator = DocIdSetBuilderIterator;

  fn iterator(&self) -> Result<Self::DocIdSetIterator> {
    match self {
      DocIdSetBuilderEnum::BitDoc(m) => Ok(DocIdSetBuilderIterator::BitSet(m.iterator()?)),
      DocIdSetBuilderEnum::IntArray(m) => Ok(DocIdSetBuilderIterator::IntArray(m.iterator()?)),
    }
  }

  type Bits = Arc<FixedBitSet>;

  fn bits(&self) -> Option<Self::Bits> {
    match self {
      DocIdSetBuilderEnum::BitDoc(bit_doc_id_set) => bit_doc_id_set.bits(),
      DocIdSetBuilderEnum::IntArray(_) => None,
    }
  }
}
pub enum DocIdSetBuilderIterator<D = Rc<Vec<i32>>, B = Arc<FixedBitSet>> {
  BitSet(BitSetIterator<B>),
  IntArray(IntArrayDocIdSetIterator<D>),
}
pub type OwnedDocIdSetBuilderIterator = DocIdSetBuilderIterator<Vec<i32>, FixedBitSet>;
impl<D: Borrow<Vec<i32>>, B: BitSet>
  crate::core::search::doc_id_set_iterator::DocIdSetIteratorExtensions
  for DocIdSetBuilderIterator<D, B>
{
}
impl<D: Borrow<Vec<i32>>, B: BitSet> BitSetIteratorAccess for DocIdSetBuilderIterator<D, B> {
  fn is_bit_iter(&self) -> bool {
    match self {
      Self::BitSet(iterator) => iterator.is_bit_iter(),
      Self::IntArray(iterator) => iterator.is_bit_iter(),
    }
  }

  fn get(&self, index: usize) -> Result<bool> {
    match self {
      Self::BitSet(iterator) => iterator.get(index),
      Self::IntArray(iterator) => iterator.get(index),
    }
  }

  fn set_doc_id(&mut self, doc: i32) -> Result<()> {
    match self {
      Self::BitSet(iterator) => BitSetIteratorAccess::set_doc_id(iterator, doc),
      Self::IntArray(iterator) => iterator.set_doc_id(doc),
    }
  }

  fn bit_set_length(&self) -> Result<usize> {
    match self {
      Self::BitSet(iterator) => iterator.bit_set_length(),
      Self::IntArray(iterator) => iterator.bit_set_length(),
    }
  }
}

impl<D: Borrow<Vec<i32>>, B: BitSet> DocIdSetIterator for DocIdSetBuilderIterator<D, B> {
  fn doc_id(&self) -> i32 {
    match self {
      DocIdSetBuilderIterator::BitSet(bit_set) => bit_set.doc_id(),
      DocIdSetBuilderIterator::IntArray(int_array) => int_array.doc_id(),
    }
  }

  fn next_doc(&mut self) -> Result<i32> {
    match self {
      DocIdSetBuilderIterator::BitSet(bit_set) => bit_set.next_doc(),
      DocIdSetBuilderIterator::IntArray(int_array) => int_array.next_doc(),
    }
  }

  fn advance(&mut self, target: i32) -> Result<i32> {
    match self {
      DocIdSetBuilderIterator::BitSet(bit_set) => bit_set.advance(target),
      DocIdSetBuilderIterator::IntArray(int_array) => int_array.advance(target),
    }
  }

  fn cost(&self) -> Result<i64> {
    match self {
      DocIdSetBuilderIterator::BitSet(bit_set) => bit_set.cost(),
      DocIdSetBuilderIterator::IntArray(int_array) => int_array.cost(),
    }
  }
}