rlucene 0.1.0-alpha.3

A Rust port of Apache Lucene, a full-text search engine library
Documentation
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
use crate::core::analysis::token_stream::TokenStream;
use crate::core::document::document::Document;
use crate::core::document::field::Field;
use crate::core::document::field_type::FieldType;
use crate::core::document::fields::FieldTokenStreamEnum;
use crate::core::document::text_field::TYPE_NOT_STORED;
use crate::core::index::concurrent_merge_scheduler::ConcurrentMergeScheduler;
use crate::core::index::index_options::IndexOptions;
use crate::core::index::index_writer::IndexWriter;
use crate::core::index::index_writer_config::{IndexWriterConfig, OpenMode};
use crate::core::index::live_index_writer_config::LiveIndexWriterConfig;
use crate::core::index::merge_policy::MergePolicyEnum;
use crate::core::util::attribute_source::{AttributeSource, Attributes};
use crate::core::util::close::{Closeable, CloseableRef};
use crate::core::util::error::lucene_error::Result;
use crate::test_framework::core::analysis::mock_analyzer::MockAnalyzer;
use crate::test_framework::core::store::mock_directory_wrapper::Throttling;
use crate::test_framework::core::util::lucene_test_case::{
  create_temp_dir_with_prefix, new_fs_directory, new_log_merge_policy_with_merge_factor_cfs, random,
};
use std::sync::Arc;

/// Tests indexing about 82 million documents with 26 terms each, producing more than
/// `i32::MAX` term/document pairs.
#[allow(dead_code)] // for quick search
struct Test2BPostings;

#[cfg(feature = "nightly")]
#[test]
#[ignore = "nightly"]
fn test() -> Result<()> {
  let mut random = random();
  let dir = new_fs_directory(&mut random, create_temp_dir_with_prefix("2BPostings")?)?;
  if let crate::core::store::directory::DirEnum::B(dir) = dir.as_ref() {
    dir.set_throttling(Throttling::Never);
  }

  let analyzer = MockAnalyzer::new(&mut random);
  let mut iwc = IndexWriterConfig::with_analyzer(analyzer)?;
  let mut merge_policy = new_log_merge_policy_with_merge_factor_cfs(&mut random, false, 10)?;
  if let MergePolicyEnum::LogBytesSize(mp) = &mut merge_policy {
    // 1 petabyte:
    mp.set_max_merge_mb(1024.0 * 1024.0 * 1024.0);
  }
  iwc
    .set_max_buffered_docs(-1)
    .set_ram_buffer_size_mb(256.0)
    .set_merge_scheduler(ConcurrentMergeScheduler::new())
    .set_merge_policy(merge_policy)
    .set_open_mode(OpenMode::Create);

  let w = IndexWriter::new(dir.clone(), iwc)?;

  let mut ft = FieldType::from_ref(&*TYPE_NOT_STORED)?;
  ft.set_omit_norms(true)?;
  ft.set_index_options(IndexOptions::Docs)?;

  let num_docs = (i32::MAX / 26) + 1;
  let mut doc = Document::new();
  doc.add(Field::from_token_stream(
    "field",
    FieldTokenStreamEnum::custom(MyTokenStream::new()),
    ft,
  )?);
  for _ in 0..num_docs {
    w.add_document(&mut doc)?;
  }
  w.force_merge(1)?;
  w.close()?;
  dir.as_ref().close()?;
  Ok(())
}

struct MyTokenStream {
  attrs: Attributes,
  index: u8,
}

impl MyTokenStream {
  fn new() -> Self {
    Self {
      attrs: Attributes::default(),
      index: b'a',
    }
  }
}

impl Closeable for MyTokenStream {}

impl TokenStream for MyTokenStream {
  fn increment_token(&mut self) -> Result<bool> {
    if self.index <= b'z' {
      self.attrs.clear_attributes()?;
      self.attrs.set_length(1)?;
      self.attrs.buffer_mut()?[0] = self.index as char;
      self.index += 1;
      return Ok(true);
    }
    Ok(false)
  }

  fn reset(&mut self) -> Result<()> {
    self.index = b'a';
    Ok(())
  }

  fn end(&mut self) -> Result<()> {
    self.default_end()
  }

  fn get_attribute_source(&self) -> &Attributes {
    &self.attrs
  }

  fn get_attribute_source_mut(&mut self) -> &mut Attributes {
    &mut self.attrs
  }
}