use crate::core::index::field_infos::FieldNumbersLock;
use crate::core::index::index_reader::IndexReader;
use crate::core::index::index_reader_context::IndexReaderContext;
use crate::core::index::index_writer::IndexWriterDir;
use crate::core::index::leaf_reader::LeafReader;
use crate::core::index::pending_deletes::{PendingDeletes, PendingDeletesEnum};
use crate::core::index::pending_soft_deletes::PendingSoftDeletes;
use crate::core::index::readers_and_updates::ReadersAndUpdates;
use crate::core::index::segment_commit_info::SegmentCommitInfo;
use crate::core::index::segment_infos::SegmentInfos;
use crate::core::index::segment_reader::SegmentReader;
use crate::core::index::sorter::DocMapImpl;
use crate::core::index::standard_directory_reader::StandardDirectoryReader;
use crate::core::store::directory::Directory;
use crate::core::util::error::lucene_error::{LuceneError, Result};
use crate::core::util::info_stream::InfoStreamMT;
use crate::core::util::long_supplier::LongSupplier;
use crate::core::util::{HasIdentity, IOUtils};
use parking_lot::Mutex;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
pub(crate) struct ReaderPool<D, F>
where
D: Directory,
F: LongSupplier,
{
directory: Arc<IndexWriterDir<D>>,
original_directory: Arc<D>,
info_stream: InfoStreamMT,
soft_deletes_field: Option<Arc<str>>,
pool_readers: AtomicBool,
inner: Mutex<Inner<D>>,
completed_del_gen_supplier: F,
index_created_version_major: i32,
}
pub(crate) struct Inner<D>
where
D: Directory,
{
reader_map: HashMap<Arc<str>, Arc<ReadersAndUpdates<D>>>,
closed: AtomicBool,
}
impl<D, F> ReaderPool<D, F>
where
D: Directory,
F: LongSupplier,
{
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
directory: Arc<IndexWriterDir<D>>,
original_directory: Arc<D>,
segment_infos: &SegmentInfos<D>,
info_stream: InfoStreamMT,
soft_deletes_field: Option<Arc<str>>,
completed_del_gen_supplier: F,
reader: Option<&StandardDirectoryReader<D>>,
index_created_version_major: i32,
) -> Result<Self> {
let mut reader_map = HashMap::with_capacity(reader.map_or(0, |_| segment_infos.size()));
if let Some(reader) = reader {
let context = reader.get_context()?;
let leaves = context.leaves()?;
debug_assert_eq!(segment_infos.size(), leaves.len());
for (i, leaf) in leaves.iter().enumerate() {
let seg_reader = leaf.reader().as_ref();
let info = segment_infos
.info(i)
.ok_or_else(|| LuceneError::illegal_state("SegmentCommitInfo missing"))?;
let new_reader = SegmentReader::new_from_reader(
info,
seg_reader,
seg_reader.get_live_docs()?,
seg_reader.get_hard_live_docs()?,
seg_reader.num_docs()?,
true,
)?;
let info_id = Arc::clone(&new_reader.original_si_id);
let pending_deletes =
Self::new_pending_deletes_with_reader(&soft_deletes_field, &new_reader, info)?;
reader_map.insert(
info_id,
Arc::new(ReadersAndUpdates::with_reader(
index_created_version_major,
Arc::new(new_reader),
info,
pending_deletes,
)?),
);
}
}
Ok(Self {
directory,
original_directory,
info_stream,
soft_deletes_field,
pool_readers: AtomicBool::new(false),
inner: Mutex::new(Inner {
reader_map,
closed: AtomicBool::new(false),
}),
completed_del_gen_supplier,
index_created_version_major,
})
}
pub(crate) fn assert_info_is_live(&self, _info: &SegmentCommitInfo<D>) -> bool {
true
}
pub(crate) fn drop(&self, info_id: &str, segment_infos: &mut SegmentInfos<D>) -> Result<bool> {
let mut inner = self.inner.lock();
if let Some(rld) = inner.reader_map.remove(info_id) {
debug_assert_eq!(info_id, rld.info_id.as_ref());
rld.drop_readers()?;
let remove_dropped_info = rld.ref_count() == 0;
if remove_dropped_info {
segment_infos.remove_dropped_segment_commit_info(info_id);
}
return Ok(true);
}
Ok(false)
}
pub(crate) fn ram_bytes_used(&self) -> i64 {
let inner = self.inner.lock();
let mut bytes: i64 = 0;
for rld in inner.reader_map.values() {
bytes += rld.ram_bytes_used.load(std::sync::atomic::Ordering::SeqCst);
}
bytes
}
pub(crate) fn any_deletions(&self, infos: &SegmentInfos<D>) -> Result<bool> {
let inner = self.inner.lock();
let mut found = 0;
for info in infos
.iter()
.iter()
.chain(infos.dropped_segment_commit_infos.values())
{
if let Some(rld) = inner.reader_map.get(info.info.get_id_key()) {
found += 1;
if rld.get_del_count(info) > 0 {
return Ok(true);
}
}
}
if found != inner.reader_map.len() {
return Err(LuceneError::illegal_state("SegmentCommitInfo missing"));
}
Ok(false)
}
pub(crate) fn enable_reader_pooling(&self) {
self
.pool_readers
.store(true, std::sync::atomic::Ordering::SeqCst);
}
pub(crate) fn is_reader_pooling_enabled(&self) -> bool {
self.pool_readers.load(std::sync::atomic::Ordering::SeqCst)
}
pub(crate) fn release(
&self,
rld: &ReadersAndUpdates<D>,
_assert_info_live: bool,
segment_infos: &mut SegmentInfos<D>,
merge_info: Option<&mut SegmentCommitInfo<D>>,
global_field_number: &FieldNumbersLock,
) -> Result<bool> {
let mut inner = self.inner.lock();
let mut changed = false;
rld.dec_ref();
if rld.ref_count() == 0 {
debug_assert!(
!inner.reader_map.contains_key(&rld.info_id),
"seg={} has refCount 0 but still unexpectedly exists in the reader pool",
rld.info_id
);
segment_infos.remove_dropped_segment_commit_info(&rld.info_id);
} else {
debug_assert!(
rld.ref_count() > 0,
"refCount={} reader={:?}",
rld.ref_count(),
rld.info_id
);
if !self.is_reader_pooling_enabled()
&& rld.ref_count() == 1
&& inner.reader_map.contains_key(&rld.info_id)
{
let info = match segment_infos.index_of_mut(&rld.info_id) {
Some(info) => Some(info),
None => merge_info,
};
let info = info.ok_or_else(|| LuceneError::illegal_state("info is None"))?;
if rld.write_live_docs(&self.directory, info)? {
changed = true;
}
if rld.write_field_updates(
&self.directory,
global_field_number,
self.completed_del_gen_supplier.get_as_long(),
self.info_stream.as_ref(),
Some(info),
)? {
changed = true;
}
if rld.get_num_dv_updates() == 0 {
rld.drop_readers()?;
if inner.reader_map.remove(&rld.info_id).is_some() {
debug_assert_eq!(rld.ref_count(), 0);
segment_infos.remove_dropped_segment_commit_info(&rld.info_id);
}
} else {
}
}
}
Ok(changed)
}
pub(crate) fn close(&self, segment_infos: &mut SegmentInfos<D>) -> Result<()> {
if self
.inner
.lock()
.closed
.compare_exchange(
false,
true,
std::sync::atomic::Ordering::SeqCst,
std::sync::atomic::Ordering::SeqCst,
)
.is_ok()
{
self.drop_all(segment_infos)?;
}
Ok(())
}
pub(crate) fn write_all_doc_values_updates(
&self,
infos: &mut SegmentInfos<D>,
global_field_number: &FieldNumbersLock,
) -> Result<bool> {
let copy: smallvec::SmallVec<[Arc<ReadersAndUpdates<D>>; 4]> = {
let inner = self.inner.lock();
let mut copy = smallvec::SmallVec::with_capacity(inner.reader_map.len());
copy.extend(inner.reader_map.values().cloned());
copy
};
let mut any = false;
for rld in copy {
any |= rld.write_field_updates(
&self.directory,
global_field_number,
self.completed_del_gen_supplier.get_as_long(),
self.info_stream.as_ref(),
infos.index_of_mut(&rld.info_id),
)?;
}
Ok(any)
}
pub(crate) fn write_doc_values_updates_for_merge(
&self,
info_ids: &[Arc<str>],
infos: &mut SegmentInfos<D>,
global_field_number: &FieldNumbersLock,
) -> Result<bool> {
let mut any = false;
for ids in info_ids {
let info = infos.index_of_mut(ids.as_ref()).ok_or_else(|| {
LuceneError::illegal_state(format!(
"could not find SegmentCommitInfo with {} in SegmentInfos",
ids
))
})?;
if let Some(rld) = self.get(info, false, None)? {
any |= rld.write_field_updates(
&self.directory,
global_field_number,
self.completed_del_gen_supplier.get_as_long(),
self.info_stream.as_ref(),
Some(info),
)?;
rld.set_is_merging();
}
}
Ok(any)
}
pub(crate) fn get_readers_by_ram(
&self,
) -> impl ExactSizeIterator<Item = Arc<ReadersAndUpdates<D>>> {
struct RamRecordingHolder<D>
where
D: Directory,
{
updates: Arc<ReadersAndUpdates<D>>,
ram_bytes_used: i64,
}
let mut holders: smallvec::SmallVec<[RamRecordingHolder<D>; 4]> = {
let inner = self.inner.lock();
inner
.reader_map
.values()
.map(|rld| RamRecordingHolder {
updates: Arc::clone(rld),
ram_bytes_used: rld.ram_bytes_used.load(std::sync::atomic::Ordering::SeqCst),
})
.collect()
};
holders.sort_unstable_by_key(|holder| std::cmp::Reverse(holder.ram_bytes_used));
holders.into_iter().map(|h| h.updates)
}
pub(crate) fn drop_all(&self, segment_infos: &mut SegmentInfos<D>) -> Result<()> {
let mut inner = self.inner.lock();
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
IOUtils::close_with(inner.reader_map.drain(), |(info_id, rld)| {
rld.drop_readers()?;
if rld.ref_count() == 0 {
segment_infos.remove_dropped_segment_commit_info(&info_id);
}
Ok(())
})
}));
debug_assert!(inner.reader_map.is_empty());
match result {
Ok(Ok(())) => Ok(()),
result => IOUtils::rethrow_always(result),
}
}
pub(crate) fn commit(
&self,
infos: &mut SegmentInfos<D>,
global_field_number: &FieldNumbersLock,
) -> Result<bool> {
let inner = self.inner.lock();
let mut at_least_one_change = false;
for info in infos.segments.iter_mut() {
if let Some(rld) = inner.reader_map.get(info.info.get_id_key()) {
debug_assert_eq!(rld.info_id.as_ref(), info.info.get_id_key());
let mut changed = rld.write_live_docs(&self.directory, info)?;
changed |= rld.write_field_updates(
&self.directory,
global_field_number,
self.completed_del_gen_supplier.get_as_long(),
self.info_stream.as_ref(),
Some(info),
)?;
if changed {
debug_assert!(self.assert_info_is_live(info));
at_least_one_change = true;
}
}
}
Ok(at_least_one_change)
}
pub(crate) fn any_doc_values_changes(&self) -> bool {
let inner = self.inner.lock();
for rld in inner.reader_map.values() {
if rld.get_num_dv_updates() != 0 {
return true;
}
}
false
}
pub(crate) fn get(
&self,
info: &SegmentCommitInfo<D>,
create: bool,
sort_map: Option<Arc<DocMapImpl>>,
) -> Result<Option<Arc<ReadersAndUpdates<D>>>> {
let mut inner = self.inner.lock();
debug_assert!(
info.info.dir.identity() == self.original_directory.identity(),
"info.dir={} vs {}",
info.info.dir,
self.original_directory
);
if inner.closed.load(std::sync::atomic::Ordering::SeqCst) {
debug_assert!(
inner.reader_map.is_empty(),
"Reader map is not empty: {:?}",
inner.reader_map.keys().collect::<Vec<_>>()
);
return Err(LuceneError::already_closed("ReaderPool is already closed"));
}
let info_id = info.info.get_id_key();
let rld = if let Some(rld) = inner.reader_map.get(info_id) {
debug_assert!(
rld.info_id.as_ref() == info_id,
"rld.info={} info={} isLive?={} ",
rld.info_id,
info,
self.assert_info_is_live(info),
);
rld.clone()
} else {
if !create {
return Ok(None);
}
let info_id = Arc::clone(&info.info.id_key);
let mut v = ReadersAndUpdates::new(
self.index_created_version_major,
Arc::clone(&info_id),
self.new_pending_deletes(info)?,
);
v.sort_map = sort_map;
let rld = Arc::new(v);
inner.reader_map.insert(info_id, Arc::clone(&rld));
rld
};
if create {
rld.inc_ref();
}
#[cfg(any(test, debug_assertions))]
debug_assert!(self.no_dups(&inner));
Ok(Some(rld))
}
fn new_pending_deletes(&self, info: &SegmentCommitInfo<D>) -> Result<PendingDeletesEnum> {
match &self.soft_deletes_field {
Some(field) => Ok(PendingDeletesEnum::Soft(PendingSoftDeletes::new(
Arc::clone(field),
info,
)?)),
None => Ok(PendingDeletesEnum::PD(PendingDeletes::new(info)?)),
}
}
fn new_pending_deletes_with_reader(
soft_deletes_field: &Option<Arc<str>>,
reader: &SegmentReader<D>,
info: &SegmentCommitInfo<D>,
) -> Result<PendingDeletesEnum> {
match soft_deletes_field {
Some(field) => Ok(PendingDeletesEnum::Soft(PendingSoftDeletes::from_reader(
Arc::clone(field),
reader,
info,
)?)),
None => Ok(PendingDeletesEnum::PD(PendingDeletes::from_reader(
reader, info,
)?)),
}
}
#[cfg(any(test, debug_assertions))]
fn no_dups(&self, inner: &Inner<D>) -> bool {
let mut seen = std::collections::HashSet::new();
for rld in inner.reader_map.keys() {
debug_assert!(!seen.contains(rld), "seen twice: {}", rld);
seen.insert(rld);
}
true
}
}
impl<D, F> Drop for ReaderPool<D, F>
where
D: Directory,
F: LongSupplier,
{
fn drop(&mut self) {
}
}