use crate::core::analysis::token_stream::TokenStream;
use crate::core::document::document::Document;
use crate::core::document::field::Field;
use crate::core::document::field_type::FieldType;
use crate::core::document::fields::FieldTokenStreamEnum;
use crate::core::document::text_field::TYPE_NOT_STORED;
use crate::core::index::concurrent_merge_scheduler::ConcurrentMergeScheduler;
use crate::core::index::index_options::IndexOptions;
use crate::core::index::index_writer::IndexWriter;
use crate::core::index::index_writer_config::{IndexWriterConfig, OpenMode};
use crate::core::index::live_index_writer_config::LiveIndexWriterConfig;
use crate::core::index::merge_policy::MergePolicyEnum;
use crate::core::util::attribute_source::{AttributeSource, Attributes};
use crate::core::util::close::{Closeable, CloseableRef};
use crate::core::util::error::lucene_error::Result;
use crate::test_framework::core::analysis::mock_analyzer::MockAnalyzer;
use crate::test_framework::core::store::mock_directory_wrapper::Throttling;
use crate::test_framework::core::util::lucene_test_case::{
create_temp_dir_with_prefix, new_fs_directory, new_log_merge_policy_with_merge_factor_cfs, random,
};
use std::sync::Arc;
#[allow(dead_code)] struct Test2BPostings;
#[cfg(feature = "nightly")]
#[test]
#[ignore = "nightly"]
fn test() -> Result<()> {
let mut random = random();
let dir = new_fs_directory(&mut random, create_temp_dir_with_prefix("2BPostings")?)?;
if let crate::core::store::directory::DirEnum::B(dir) = dir.as_ref() {
dir.set_throttling(Throttling::Never);
}
let analyzer = MockAnalyzer::new(&mut random);
let mut iwc = IndexWriterConfig::with_analyzer(analyzer)?;
let mut merge_policy = new_log_merge_policy_with_merge_factor_cfs(&mut random, false, 10)?;
if let MergePolicyEnum::LogBytesSize(mp) = &mut merge_policy {
mp.set_max_merge_mb(1024.0 * 1024.0 * 1024.0);
}
iwc
.set_max_buffered_docs(-1)
.set_ram_buffer_size_mb(256.0)
.set_merge_scheduler(ConcurrentMergeScheduler::new())
.set_merge_policy(merge_policy)
.set_open_mode(OpenMode::Create);
let w = IndexWriter::new(dir.clone(), iwc)?;
let mut ft = FieldType::from_ref(&*TYPE_NOT_STORED)?;
ft.set_omit_norms(true)?;
ft.set_index_options(IndexOptions::Docs)?;
let num_docs = (i32::MAX / 26) + 1;
let mut doc = Document::new();
doc.add(Field::from_token_stream(
"field",
FieldTokenStreamEnum::custom(MyTokenStream::new()),
ft,
)?);
for _ in 0..num_docs {
w.add_document(&mut doc)?;
}
w.force_merge(1)?;
w.close()?;
dir.as_ref().close()?;
Ok(())
}
struct MyTokenStream {
attrs: Attributes,
index: u8,
}
impl MyTokenStream {
fn new() -> Self {
Self {
attrs: Attributes::default(),
index: b'a',
}
}
}
impl Closeable for MyTokenStream {}
impl TokenStream for MyTokenStream {
fn increment_token(&mut self) -> Result<bool> {
if self.index <= b'z' {
self.attrs.clear_attributes()?;
self.attrs.set_length(1)?;
self.attrs.buffer_mut()?[0] = self.index as char;
self.index += 1;
return Ok(true);
}
Ok(false)
}
fn reset(&mut self) -> Result<()> {
self.index = b'a';
Ok(())
}
fn end(&mut self) -> Result<()> {
self.default_end()
}
fn get_attribute_source(&self) -> &Attributes {
&self.attrs
}
fn get_attribute_source_mut(&mut self) -> &mut Attributes {
&mut self.attrs
}
}