use crate::core::index::index_commit::{IndexCommit, cmp_commit, is_same_commit};
use crate::core::index::index_deletion_policy::IndexDeletionPolicy;
#[cfg(test)]
use crate::core::index::index_writer::IndexWriter;
use crate::core::index::index_writer::IndexWriterDir;
use crate::core::index::segment_infos::SegmentInfos;
use crate::core::index::{CODEC_FILE_PATTERN, IndexFileNames};
use crate::core::store::directory::Directory;
use crate::core::util::IOUtils;
use crate::core::util::close::Closeable;
use crate::core::util::error::lucene_error::{
CaughtResult, CaughtResultExt, LuceneError, LuceneErrorKind, Result,
};
use crate::core::util::file_deleter::{FileDeleter, Messenger, MsgType};
use crate::core::util::info_stream::{InfoStream, InfoStreamMT};
#[cfg(test)]
use crate::test_framework::core::util::failure_context::{
ExecutionMethod, ExecutionOwner, ExecutionScope,
};
use std::cmp::Ordering;
use std::collections::{HashMap, HashSet};
use std::fmt::{Display, Formatter};
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering::SeqCst;
pub struct IndexFileDeleter<D>
where
D: Directory,
{
commits: Vec<Arc<CommitPoint<D>>>,
last_files: Option<CheckpointFiles>,
commits_to_delete: Arc<AtomicBool>,
info_stream: InfoStreamMT,
directory_orig: Arc<D>,
directory: Arc<IndexWriterDir<D>>,
pub(crate) starting_commit_deleted: bool,
file_deleter: FileDeleter<D, MessengerImpl>,
}
impl<D> IndexFileDeleter<D>
where
D: Directory,
{
#[allow(clippy::too_many_arguments)]
pub fn new<P, I>(
files: I,
directory_orig: Arc<D>,
directory: Arc<IndexWriterDir<D>>,
policy: &P,
segment_infos: &mut SegmentInfos<D>,
info_stream: InfoStreamMT,
initial_index_exists: bool,
is_reader_init: bool,
) -> Result<Self>
where
P: IndexDeletionPolicy<Arc<CommitPoint<D>>>,
I: IntoIterator<Item = String>,
{
let files = files.into_iter();
let commits = Vec::new();
let mut last_segment_infos: Option<SegmentInfos<D>> = None;
let current_segments_file_opt = segment_infos.get_segments_file_name();
if info_stream.is_enabled("IFD") {
info_stream.message(
"IFD",
&format!(
"init: current segments file is \"{current_segments_file_opt:?}\"; deletionPolicy={policy}"
),
)?;
}
let file_deleter = FileDeleter::new(
directory.clone(),
Some(MessengerImpl::new(info_stream.clone(), false)),
);
let mut index_file_deleter = Self {
commits,
last_files: None,
commits_to_delete: Arc::new(AtomicBool::new(false)),
info_stream,
directory_orig: directory_orig.clone(),
directory,
starting_commit_deleted: !initial_index_exists,
file_deleter,
};
let mut current_commit_point = None;
if current_segments_file_opt.is_some() {
let current_gen = segment_infos.get_generation();
for file in files {
if file.ends_with("write.lock") {
continue;
}
let is_segments = file.starts_with(IndexFileNames::SEGMENTS);
if CODEC_FILE_PATTERN.is_match(&file)
|| is_segments
|| file.starts_with(IndexFileNames::PENDING_SEGMENTS)
{
if is_segments {
index_file_deleter.file_deleter.init_ref_count(&file);
if index_file_deleter.info_stream.is_enabled("IFD") {
index_file_deleter
.info_stream
.message("IFD", &format!("init: load commit \"{file}\""))?;
}
let sis = SegmentInfos::read_commit(directory_orig.clone(), &file)?;
let commit_point = Arc::new(CommitPoint::new(
index_file_deleter.commits_to_delete.clone(),
directory_orig.clone(),
&sis,
None,
)?);
if sis.get_generation() == current_gen {
current_commit_point = Some(Arc::clone(&commit_point));
}
index_file_deleter.commits.push(Arc::clone(&commit_point));
index_file_deleter.inc_ref_files(commit_point.files.iter())?;
if last_segment_infos
.as_ref()
.is_none_or(|last| sis.get_generation() > last.get_generation())
{
last_segment_infos = Some(sis);
}
} else {
index_file_deleter.file_deleter.init_ref_count_owned(file);
}
}
}
}
if let Some(file) = current_segments_file_opt
&& current_commit_point.is_none()
&& initial_index_exists
{
let sis = SegmentInfos::read_commit(directory_orig.clone(), &file);
let sis = sis.map_err(|e| {
if e.is_io_error() {
let mut error = LuceneError::corrupt_index(format!(
"unable to read current segments_N file (resource={file})"
));
error.add_suppressed(e);
error
} else {
e
}
})?;
if index_file_deleter.info_stream.is_enabled("IFD") {
index_file_deleter.info_stream.message(
"IFD",
&format!(
"forced open of current segments file {:?}",
segment_infos.get_segments_file_name()
),
)?;
}
let commit_point = Arc::new(CommitPoint::new(
index_file_deleter.commits_to_delete.clone(),
directory_orig.clone(),
&sis,
None,
)?);
current_commit_point = Some(Arc::clone(&commit_point));
index_file_deleter.commits.push(Arc::clone(&commit_point));
index_file_deleter.inc_ref_files(commit_point.files.iter())?;
}
if is_reader_init {
index_file_deleter.checkpoint(segment_infos, false, policy)?;
}
index_file_deleter.commits.sort_unstable();
let pending = directory_orig.get_pending_deletions()?;
let relevant_files = index_file_deleter.file_deleter.get_all_files();
if !pending.is_empty() {
let relevant_files = relevant_files.chain(pending.iter());
inflate_gens(
segment_infos,
relevant_files,
&index_file_deleter.info_stream,
)?;
} else {
inflate_gens(
segment_infos,
relevant_files,
&index_file_deleter.info_stream,
)?;
}
let unrefed = index_file_deleter.file_deleter.get_unrefed_files()?;
for file in &unrefed {
if file.starts_with(IndexFileNames::SEGMENTS) {
return Err(LuceneError::illegal_state(
"file \"{file}\" has refCount=0, which should never happen on init",
));
}
if index_file_deleter.info_stream.is_enabled("IFD") {
index_file_deleter.info_stream.message(
"IFD",
&format!("init: removing unreferenced file \"{file}\""),
)?;
}
}
index_file_deleter
.file_deleter
.delete_owned_files_if_no_ref(unrefed)?;
policy.on_init(&index_file_deleter.commits)?;
index_file_deleter.checkpoint(segment_infos, false, policy)?;
index_file_deleter.starting_commit_deleted = match current_commit_point {
Some(commit) => commit.is_deleted(),
None => false,
};
index_file_deleter.delete_commits()?;
Ok(index_file_deleter)
}
#[cfg(test)]
fn ensure_open(&self, index_writer: &IndexWriter<D>) -> Result<()> {
index_writer.do_ensure_open(false)?;
let tragic_arc = index_writer.get_tragic_exception();
let error = tragic_arc.get();
if let Some(e) = error {
let mut error = LuceneError::already_closed(
"refusing to delete any files: this IndexWriter encountered an unrecoverable error",
);
error.add_suppressed(e.clone());
return Err(error);
}
Ok(())
}
#[cfg(test)]
pub(crate) fn is_closed(&self, index_writer: &IndexWriter<D>) -> Result<bool> {
match self.ensure_open(index_writer) {
Ok(_) => Ok(false),
Err(e) => {
if matches!(e.kind(), LuceneErrorKind::AlreadyClosed(_)) {
Ok(true)
} else {
Err(e)
}
},
}
}
fn delete_commits(&mut self) -> Result<()> {
#[cfg(test)]
let _execution_scope = ExecutionScope::enter(
ExecutionOwner::IndexFileDeleter,
ExecutionMethod::DeleteCommits,
);
if !self.commits_to_delete.load(SeqCst) {
return Ok(());
}
let mut removed = smallvec::SmallVec::<[_; 2]>::new();
for commit in &self.commits {
if commit.is_deleted() {
removed.push(commit);
}
}
let mut first_failure: Option<CaughtResult> = None;
for commit in removed {
if self.info_stream.is_enabled("IFD") {
self.info_stream.message(
"IFD",
&format!(
"deleteCommits: now decRef commit \"{}\"",
commit.get_segments_file_name()
),
)?;
}
let failure = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
#[cfg(test)]
let _execution_scope =
ExecutionScope::enter(ExecutionOwner::IndexFileDeleter, ExecutionMethod::DecRef);
self.file_deleter.dec_ref(commit.files.iter())
}));
if !matches!(&failure, Ok(Ok(()))) {
match first_failure.as_mut() {
Some(first_failure) => {
first_failure.add_suppressed(failure, "panic while decrementing file references")
},
None => first_failure = Some(failure),
}
}
}
self.commits_to_delete.store(false, SeqCst);
self.commits.retain(|commit| !commit.is_deleted());
if let Some(first_failure) = first_failure {
return IOUtils::rethrow_always(first_failure);
}
Ok(())
}
pub(crate) fn refresh(&self) -> Result<()> {
let mut to_delete = HashSet::new();
let files = self.directory.list_all()?;
for file_name in files {
let is_lock_file = file_name.ends_with("write.lock");
let is_codec_match = CODEC_FILE_PATTERN.is_match(&file_name);
let is_segments = file_name.starts_with(IndexFileNames::SEGMENTS);
let is_pending_segments = file_name.starts_with(IndexFileNames::PENDING_SEGMENTS);
if !is_lock_file
&& !self.file_deleter.exists(&file_name)
&& (is_codec_match || is_segments || is_pending_segments)
{
if self.info_stream.is_enabled("IFD") {
self.info_stream.message(
"IFD",
&format!("refresh: removing newly created unreferenced file \"{file_name}\""),
)?;
}
to_delete.insert(file_name);
}
}
self.file_deleter.delete_owned_files_if_no_ref(to_delete)
}
fn assert_commits_are_not_deleted(&self, commits: &[Arc<CommitPoint<D>>]) -> bool {
for commit in commits {
debug_assert!(
!commit.is_deleted(),
"Commit [{commit}] was deleted already"
);
}
true
}
pub(crate) fn revisit_policy<P>(&mut self, policy: &P) -> Result<()>
where
P: IndexDeletionPolicy<Arc<CommitPoint<D>>>,
{
{
if self.info_stream.is_enabled("IFD") {
self.info_stream.message("IFD", "now revisitPolicy")?;
}
}
if !self.commits.is_empty() {
debug_assert!(self.assert_commits_are_not_deleted(&self.commits));
policy.on_commit(&self.commits)?;
self.delete_commits()?;
}
Ok(())
}
pub fn checkpoint<P>(
&mut self,
segment_infos: &SegmentInfos<D>,
is_commit: bool,
policy: &P,
) -> Result<()>
where
P: IndexDeletionPolicy<Arc<CommitPoint<D>>>,
{
#[cfg(test)]
let _execution_scope = ExecutionScope::enter(
ExecutionOwner::IndexFileDeleter,
ExecutionMethod::Checkpoint,
);
let t0 = std::time::Instant::now();
{
if self.info_stream.is_enabled("IFD") {
}
}
if is_commit {
let files = segment_infos.files(true)?;
self.inc_ref_files(files.iter())?;
self.commits.push(Arc::new(CommitPoint::new(
Arc::clone(&self.commits_to_delete),
Arc::clone(&self.directory_orig),
segment_infos,
Some(files),
)?));
debug_assert!(self.assert_commits_are_not_deleted(&self.commits));
policy.on_commit(&self.commits)?;
self.delete_commits()?;
} else {
let new_files = if let [info] = segment_infos.iter()
&& info.has_only_base_files()
{
CheckpointFiles::Shared(Arc::clone(info.info.files_snapshot()?))
} else {
CheckpointFiles::Owned(segment_infos.files(false)?)
};
self.inc_ref_files(new_files.as_set().iter())?;
let old_files = self.last_files.take();
self.dec_ref(old_files.iter().flat_map(|files| files.as_set().iter()))?;
drop(old_files);
self.last_files = Some(new_files);
}
{
if self.info_stream.is_enabled("IFD") {
let elapsed_ms = t0.elapsed().as_millis();
self
.info_stream
.message("IFD", &format!("{elapsed_ms} ms to checkpoint"))?;
}
}
Ok(())
}
pub fn inc_ref_from_segment(
&mut self,
segment_infos: &SegmentInfos<D>,
is_commit: bool,
) -> Result<()> {
let files = segment_infos.files_for_read(is_commit)?;
self.inc_ref_files(files.iter())
}
pub fn inc_ref_files<I>(&mut self, files: I) -> Result<()>
where
I: IntoIterator,
I::Item: AsRef<str>,
{
self.file_deleter.inc_ref(files)
}
pub(crate) fn dec_ref<'a, I>(&mut self, files: I) -> Result<()>
where
I: IntoIterator<Item = &'a String>,
{
#[cfg(test)]
let _execution_scope =
ExecutionScope::enter(ExecutionOwner::IndexFileDeleter, ExecutionMethod::DecRef);
self.file_deleter.dec_ref(files)
}
pub(crate) fn dec_ref_from_segment(&mut self, segment_infos: &SegmentInfos<D>) -> Result<()> {
let files = segment_infos.files_for_read(false)?;
self.dec_ref(files.iter())
}
pub fn exists(&self, file_name: &str) -> bool {
self.file_deleter.exists(file_name)
}
pub(crate) fn delete_new_files<'a, I>(&self, files: I) -> Result<()>
where
I: IntoIterator<Item = &'a String>,
{
self.file_deleter.delete_files_if_no_ref(files)
}
}
pub(crate) enum CheckpointFiles {
Shared(Arc<HashSet<String>>),
Owned(HashSet<String>),
}
impl CheckpointFiles {
pub(crate) fn as_set(&self) -> &HashSet<String> {
match self {
Self::Shared(files) => files,
Self::Owned(files) => files,
}
}
}
impl<D> Closeable for IndexFileDeleter<D>
where
D: Directory,
{
fn close(&mut self) -> Result<()> {
if let Some(files) = self.last_files.take()
&& !files.as_set().is_empty()
{
self.dec_ref(files.as_set().iter())?;
}
Ok(())
}
}
pub struct CommitPoint<D> {
pub(crate) files: Vec<String>,
pub(crate) segments_file_name: String,
pub(crate) deleted: AtomicBool,
pub(crate) directory_orig: Arc<D>,
pub(crate) generation: i64,
pub(crate) user_data: HashMap<String, String>,
pub(crate) segment_count: usize,
pub(crate) commits_to_delete: Arc<AtomicBool>,
}
impl<D> CommitPoint<D>
where
D: Directory,
{
pub(crate) fn new(
commits_to_delete: Arc<AtomicBool>,
directory_orig: Arc<D>,
segment_infos: &SegmentInfos<D>,
referenced_files: Option<HashSet<String>>,
) -> Result<Self> {
let user_data = segment_infos.get_user_data().clone();
let segments_file_name = segment_infos
.get_segments_file_name()
.ok_or_else(|| LuceneError::illegal_state("segment_N file is none"))?;
let generation = segment_infos.get_generation();
let files = match referenced_files {
Some(files) => files,
None => segment_infos.files(true)?,
}
.into_iter()
.collect();
let segment_count = segment_infos.size();
Ok(CommitPoint {
files,
segments_file_name,
deleted: AtomicBool::new(false),
directory_orig,
generation,
user_data,
segment_count,
commits_to_delete,
})
}
}
impl<D> PartialEq for CommitPoint<D>
where
D: Directory,
{
fn eq(&self, other: &Self) -> bool {
is_same_commit(self, other)
}
}
impl<D> Eq for CommitPoint<D> where D: Directory {}
impl<D> PartialOrd for CommitPoint<D>
where
D: Directory,
{
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl<D> Ord for CommitPoint<D>
where
D: Directory,
{
fn cmp(&self, other: &Self) -> Ordering {
cmp_commit(self, other)
}
}
impl<D> Display for CommitPoint<D> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}({})",
std::any::type_name::<Self>(),
self.segments_file_name
)
}
}
impl<D> IndexCommit for CommitPoint<D>
where
D: Directory,
{
fn get_segments_file_name(&self) -> &str {
&self.segments_file_name
}
fn get_file_names(&self) -> Result<&[String]> {
Ok(self.files.as_slice())
}
type Directory = Arc<D>;
fn get_directory(&self) -> Self::Directory {
self.directory_orig.clone()
}
fn delete(&self) -> Result<()> {
if !self.deleted.swap(true, SeqCst) {
self
.commits_to_delete
.store(true, std::sync::atomic::Ordering::SeqCst);
}
Ok(())
}
fn is_deleted(&self) -> bool {
self.deleted.load(SeqCst)
}
fn get_segment_count(&self) -> usize {
self.segment_count
}
fn get_generation(&self) -> i64 {
self.generation
}
fn get_user_data(&self) -> &HashMap<String, String> {
&self.user_data
}
}
pub(crate) struct MessengerImpl {
info_stream: InfoStreamMT,
verbose_ref_counts: bool,
}
impl MessengerImpl {
pub(crate) fn new(info_stream: InfoStreamMT, verbose_ref_counts: bool) -> Self {
MessengerImpl {
info_stream,
verbose_ref_counts,
}
}
}
impl Messenger for MessengerImpl {
fn accept(&self, msg_type: MsgType, msg: &str) -> Result<()> {
if msg_type == MsgType::Ref && !self.verbose_ref_counts {
return Ok(());
}
if self.info_stream.is_enabled("IFD") {
self.info_stream.message("IFD", msg)?;
}
Ok(())
}
fn accept_formatted(&self, msg_type: MsgType, msg: std::fmt::Arguments<'_>) -> Result<()> {
if msg_type == MsgType::Ref && !self.verbose_ref_counts {
return Ok(());
}
if self.info_stream.is_enabled("IFD") {
self.info_stream.message("IFD", &msg.to_string())?;
}
Ok(())
}
}
use crate::core::index::index_writer::WRITE_LOCK_NAME;
use crate::core::index::segment_infos::generation_from_segments_file_name;
pub(crate) fn inflate_gens<'a, D, I>(
infos: &mut SegmentInfos<D>,
files: I,
info_stream: &InfoStreamMT,
) -> Result<()>
where
I: IntoIterator<Item = &'a String>,
{
let mut max_segment_gen = i64::MIN;
let mut max_segment_name = i64::MIN;
let mut max_per_segment_gen = HashMap::with_capacity(infos.size());
for file_name in files {
if file_name == WRITE_LOCK_NAME {
continue;
} else if file_name.starts_with(IndexFileNames::SEGMENTS) {
let v = generation_from_segments_file_name(file_name);
match v {
Ok(gen_) => {
max_segment_gen = max_segment_gen.max(gen_);
},
Err(e) => {
if !matches!(e.kind(), LuceneErrorKind::NumberFormat(_)) {
return Err(e);
}
},
}
} else if file_name.starts_with(IndexFileNames::PENDING_SEGMENTS) {
let v = generation_from_segments_file_name(&file_name[8..]);
match v {
Ok(gen_) => {
max_segment_gen = max_segment_gen.max(gen_);
},
Err(e) => {
if !matches!(e.kind(), LuceneErrorKind::NumberFormat(_)) {
return Err(e);
}
},
}
} else {
let segment_name = IndexFileNames::parse_segment_name(file_name);
debug_assert!(segment_name.starts_with('_'), "wtf? file={file_name}");
let is_temp_file = if file_name.is_ascii() {
file_name
.get(file_name.len().saturating_sub(4)..)
.is_some_and(|suffix| suffix.eq_ignore_ascii_case(".tmp"))
} else {
file_name.to_lowercase().ends_with(".tmp")
};
if is_temp_file {
continue;
}
max_segment_name = max_segment_name.max(i64::from_str_radix(&segment_name[1..], 36)?);
let mut cur_gen = *max_per_segment_gen.get(segment_name).unwrap_or(&0i64);
let v = IndexFileNames::parse_generation(file_name);
match v {
Ok(gen_) => {
cur_gen = cur_gen.max(gen_);
},
Err(e) => {
if !matches!(e.kind(), LuceneErrorKind::NumberFormat(_)) {
return Err(e);
}
},
}
max_per_segment_gen.insert(segment_name, cur_gen);
}
}
let next_gen = infos.get_generation().max(max_segment_gen);
infos.set_next_write_generation(next_gen)?;
let desired = 1 + max_segment_name;
if infos.counter < desired {
if info_stream.is_enabled("IFD") {
info_stream.message(
"IFD",
&format!(
"init: inflate infos.counter to {} vs current={}",
desired, infos.counter
),
)?;
}
infos.counter = desired;
}
for info in infos.iter_mut() {
let gen_long = *max_per_segment_gen
.get(info.info.name.as_ref())
.ok_or_else(|| {
LuceneError::illegal_state(format!(
"missing maximum generation for segment {}",
info.info.name
))
})?;
let next_del = info.get_next_write_del_gen();
if next_del < gen_long + 1 {
if info_stream.is_enabled("IFD") {
info_stream.message(
"IFD",
&format!(
"init: seg={} set nextWriteDelGen={} vs current={}",
info.info.name,
gen_long + 1,
next_del
),
)?;
}
info.set_next_write_del_gen(gen_long + 1);
}
let next_fi = info.get_next_write_field_infos_gen();
if next_fi < gen_long + 1 {
if info_stream.is_enabled("IFD") {
info_stream.message(
"IFD",
&format!(
"init: seg={} set nextWriteFieldInfosGen={} vs current={}",
info.info.name,
gen_long + 1,
next_fi
),
)?;
}
info.set_next_write_field_infos_gen(gen_long + 1);
}
let next_dv = info.get_next_write_doc_values_gen();
if next_dv < gen_long + 1 {
if info_stream.is_enabled("IFD") {
info_stream.message(
"IFD",
&format!(
"init: seg={} set nextWriteDocValuesGen={} vs current={}",
info.info.name,
gen_long + 1,
next_dv
),
)?;
}
info.set_next_write_doc_values_gen(gen_long + 1);
}
}
Ok(())
}