use crate::core::index::frozen_buffered_updates::FrozenBufferedUpdates;
use crate::core::index::index_writer::IndexWriter;
use crate::core::index::readers_and_updates;
use crate::core::index::readers_and_updates::ReadersAndUpdates;
use crate::core::index::segment_commit_info::SegmentCommitInfo;
use crate::core::index::segment_reader::DefaultLeafReader;
use crate::core::store::IO_CONTEXT_DEFAULT;
use crate::core::store::directory::Directory;
use crate::core::util::IOUtils;
use crate::core::util::accountable::Accountable;
use crate::core::util::error::lucene_error::{LuceneError, Result};
use crate::core::util::info_stream::{InfoStream, InfoStreamMT};
use parking_lot::Mutex;
use std::collections::{HashMap, HashSet};
use std::fmt::Display;
use std::sync::Arc;
use std::sync::atomic::{AtomicI64, Ordering};
pub(crate) struct BufferedUpdatesStream {
inner: Mutex<BufferedUpdatesStreamInner>,
bytes_used: AtomicI64,
finished_segments: FinishedSegments,
}
pub(crate) struct BufferedUpdatesStreamInner {
updates: HashMap<usize, Arc<FrozenBufferedUpdates>>,
next_gen: i64,
}
impl BufferedUpdatesStream {
pub(crate) fn new(info_stream: InfoStreamMT) -> Self {
Self {
inner: Mutex::new(BufferedUpdatesStreamInner {
updates: HashMap::new(),
next_gen: 1,
}),
bytes_used: AtomicI64::new(0),
finished_segments: FinishedSegments::new(info_stream),
}
}
pub(crate) fn push(
&self,
mut packet: FrozenBufferedUpdates,
) -> Result<(i64, Arc<FrozenBufferedUpdates>)> {
let mut inner = self.inner.lock();
packet.set_del_gen(inner.next_gen);
inner.next_gen += 1;
debug_assert!(packet.any());
debug_assert!(self.check_delete_stats(&inner));
let bytes_used = packet.bytes_used;
let del_gen = packet.del_gen();
let v = Arc::new(packet);
inner.updates.insert(Arc::as_ptr(&v).addr(), Arc::clone(&v));
self
.bytes_used
.fetch_add(bytes_used as i64, Ordering::SeqCst);
{
if self.finished_segments.info_stream.is_enabled("BD") {
let count = inner.updates.len();
let used_mb = self.bytes_used.load(Ordering::SeqCst) as f64 / 1024.0 / 1024.0;
self.finished_segments.info_stream.message(
"BD",
&format!("push new packet ({v}), packetCount={count}, bytesUsed={used_mb:.3} MB"),
)?;
}
}
debug_assert!(self.check_delete_stats(&inner));
Ok((del_gen, v))
}
pub(crate) fn get_pending_updates_count(&self) -> usize {
let inner = self.inner.lock();
inner.updates.len()
}
#[allow(dead_code)]
pub(crate) fn clear(&self) {
let mut inner = self.inner.lock();
inner.updates.clear();
inner.next_gen = 1;
self.finished_segments.clear();
self.bytes_used.store(0, Ordering::SeqCst);
}
pub(crate) fn any(&self) -> bool {
self.bytes_used.load(Ordering::SeqCst) != 0
}
pub(crate) fn wait_apply_all<D>(&self, writer: &IndexWriter<D>) -> Result<()>
where
D: Directory + 'static,
{
let wait_for = {
let inner = self.inner.lock();
inner.updates.values().cloned().collect()
};
self.wait_apply(wait_for, writer)
}
pub(crate) fn still_running(&self, del_gen: i64) -> bool {
self.finished_segments.still_running(del_gen)
}
pub(crate) fn finished_segment(&self, del_gen: i64) -> Result<()> {
self.finished_segments.finished_segment(del_gen)
}
pub(crate) fn finished(&self, packet: &FrozenBufferedUpdates) -> Result<()> {
debug_assert!(!packet.applied.load(Ordering::SeqCst), "packet={packet}");
packet.applied.store(true, Ordering::SeqCst);
let mut inner = self.inner.lock();
inner.updates.remove(&std::ptr::from_ref(packet).addr());
let bytes = packet.bytes_used as i64;
self.bytes_used.fetch_sub(bytes, Ordering::SeqCst);
self.finished_segment(packet.del_gen())?;
Ok(())
}
pub fn get_completed_del_gen(&self) -> i64 {
self.finished_segments.get_completed_del_gen()
}
pub(crate) fn wait_apply_for_merge<D>(
&self,
merge_infos_id: &[Arc<str>],
writer: &IndexWriter<D>,
) -> Result<()>
where
D: Directory + 'static,
{
let mut max_del_gen = i64::MIN;
{
let writer_inner = writer.inner.lock();
for info in merge_infos_id {
let info = writer_inner
.segment_infos
.index_of(info.as_ref())
.ok_or_else(|| {
LuceneError::illegal_argument(
"could not find merge's segment from IndexWriter's segment_infos",
)
})?;
max_del_gen = max_del_gen.max(info.get_buffered_deletes_gen());
}
}
let wait_for = {
let inner = self.inner.lock();
let mut packets = smallvec::SmallVec::new();
for packet in inner.updates.values() {
if packet.del_gen() <= max_del_gen {
packets.push(packet.clone());
}
}
packets
};
if self.finished_segments.info_stream.is_enabled("BD") {
self.finished_segments.info_stream.message(
"BD",
&format!(
"waitApplyForMerge: {} packets, {} merging segments",
wait_for.len(),
merge_infos_id.len()
),
)?;
}
self.wait_apply(wait_for, writer)
}
fn wait_apply<D>(
&self,
mut wait_for: smallvec::SmallVec<[Arc<FrozenBufferedUpdates>; 4]>,
writer: &IndexWriter<D>,
) -> Result<()>
where
D: Directory + 'static,
{
let start_ns = std::time::Instant::now();
let packet_count = wait_for.len();
if wait_for.is_empty() {
if self.finished_segments.info_stream.is_enabled("BD") {
self
.finished_segments
.info_stream
.message("BD", "waitApply: no deletes to apply")?;
}
return Ok(());
}
if self.finished_segments.info_stream.is_enabled("BD") {
self.finished_segments.info_stream.message(
"BD",
&format!("waitApply: {packet_count:?} packets: {wait_for:?}"),
)?;
}
let mut pending_count = 0;
let mut total_del_count: i64 = 0;
for index in 0..wait_for.len() {
let packet = &wait_for[index];
let applied = writer.try_apply(packet)?;
total_del_count += packet.total_del_count.load(Ordering::SeqCst);
if !applied {
wait_for.swap(pending_count, index);
pending_count += 1;
}
}
for packet in &wait_for[..pending_count] {
writer.force_apply(packet)?;
}
if self.finished_segments.info_stream.is_enabled("BD") {
let elapsed = start_ns.elapsed().as_secs_f64() * 1000.0;
let bytes = self.bytes_used.load(Ordering::SeqCst);
self.finished_segments.info_stream.message(
"BD",
&format!(
"waitApply: done {packet_count} packets; totalDelCount={total_del_count}; totBytesUsed={bytes}; took {elapsed:.2} msec"
),
)?;
}
Ok(())
}
pub(crate) fn get_next_gen(&self) -> i64 {
let mut inner = self.inner.lock();
let gen_ = inner.next_gen;
inner.next_gen += 1;
gen_
}
fn check_delete_stats(&self, inner: &BufferedUpdatesStreamInner) -> bool {
let mut bytes_used2 = 0i64;
for packet in inner.updates.values() {
bytes_used2 += packet.bytes_used as i64;
}
let actual = self.bytes_used.load(Ordering::SeqCst);
debug_assert_eq!(
bytes_used2, actual,
"bytes_used2={bytes_used2} vs bytes_used={actual}"
);
true
}
}
impl Accountable for BufferedUpdatesStream {
fn ram_bytes_used(&self) -> Result<i64> {
Ok(self.bytes_used.load(Ordering::SeqCst))
}
}
pub(crate) struct SegmentState<D>
where
D: Directory,
{
pub(crate) del_gen: i64,
pub(crate) rld: Arc<ReadersAndUpdates<D>>,
pub(crate) reader: DefaultLeafReader<D>,
pub(crate) start_del_count: i32,
}
impl<D> SegmentState<D>
where
D: Directory,
{
pub(crate) fn new(rld: Arc<ReadersAndUpdates<D>>, info: &SegmentCommitInfo<D>) -> Result<Self> {
let reader = {
let mut inner = rld.inner.lock();
readers_and_updates::get_reader(
IO_CONTEXT_DEFAULT.as_ref().map_err(Clone::clone)?,
info,
&mut inner,
rld.index_created_version_major,
)?
};
Ok(SegmentState {
del_gen: info.get_buffered_deletes_gen(),
rld,
reader,
start_del_count: info.get_del_count(),
})
}
pub(crate) fn close(
&self,
writer: &IndexWriter<D>,
inner: &mut crate::core::index::index_writer::Inner<D>,
) -> Result<()> {
IOUtils::close_with(0..2, |operation| match operation {
0 => self.rld.release(self.reader.as_ref(), None),
_ => writer.release(self.rld.as_ref(), inner),
})
}
}
impl<D> Display for SegmentState<D>
where
D: Directory,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "SegmentState({})", self.rld.info_id)
}
}
pub(crate) struct FinishedSegments {
info_stream: InfoStreamMT,
inner: Mutex<Inner>,
}
pub(crate) struct Inner {
completed_del_gen: i64,
finished_del_gens: HashSet<i64>,
}
impl FinishedSegments {
pub(crate) fn new(info_stream: InfoStreamMT) -> Self {
FinishedSegments {
info_stream,
inner: Mutex::new(Inner {
completed_del_gen: 0,
finished_del_gens: HashSet::new(),
}),
}
}
#[allow(dead_code)]
pub(crate) fn clear(&self) {
let mut inner = self.inner.lock();
inner.finished_del_gens.clear();
inner.completed_del_gen = 0;
}
pub(crate) fn still_running(&self, del_gen: i64) -> bool {
let inner = self.inner.lock();
del_gen > inner.completed_del_gen && !inner.finished_del_gens.contains(&del_gen)
}
pub fn get_completed_del_gen(&self) -> i64 {
let inner = self.inner.lock();
inner.completed_del_gen
}
pub(crate) fn finished_segment(&self, del_gen: i64) -> Result<()> {
let mut inner = self.inner.lock();
inner.finished_del_gens.insert(del_gen);
while inner
.finished_del_gens
.contains(&(inner.completed_del_gen + 1))
{
let v = inner.completed_del_gen + 1;
inner.finished_del_gens.remove(&v);
inner.completed_del_gen += 1;
}
{
if self.info_stream.is_enabled("BD") {
self.info_stream.message(
"BD",
&format!(
"finished packet delGen={} now completedDelGen={}",
del_gen, inner.completed_del_gen
),
)?;
}
}
Ok(())
}
}
pub(crate) struct ApplyDeletesResult {
pub(crate) any_deletes: bool,
pub(crate) all_deleted: Option<smallvec::SmallVec<[Arc<str>; 1]>>,
}
impl ApplyDeletesResult {
pub(crate) fn any_deletes(&self) -> bool {
self.any_deletes
}
pub(crate) fn all_deleted(&self) -> Option<&[Arc<str>]> {
self.all_deleted.as_deref()
}
}