use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::fmt;
use std::sync::Arc;
use parking_lot::Mutex;
use crate::core::codecs::Codecs;
use crate::core::codecs::codec;
use crate::core::codecs::segment_info_format::SegmentInfoFormat;
use crate::core::codecs::{Codec, CodecUtil};
use crate::core::index::IndexFileNames;
use crate::core::index::index_commit::IndexCommit;
use crate::core::index::index_file_names::base36;
use crate::core::index::index_writer::get_actual_max_docs;
use crate::core::index::segment_commit_info::SegmentCommitInfo;
use crate::core::store::check_sum_index_input::ChecksumIndexInput;
use crate::core::store::directory::Directory;
use crate::core::store::{DataInput, IO_CONTEXT_DEFAULT, IndexOutput};
use crate::core::util::close::{Closeable, CloseableRef};
use crate::core::util::error::lucene_error::{LuceneError, LuceneErrorKind, Result};
use crate::core::util::output_enum::OutputEnum;
use crate::core::util::{
HasIdentity, IOUtils, LATEST, MIN_SUPPORTED_MAJOR, StringHelper, TryIntoInt, Version,
};
#[cfg(test)]
use crate::test_framework::core::util::failure_context::{
ExecutionMethod, ExecutionOwner, ExecutionScope,
};
use std::io::{Error, ErrorKind, Write};
static INFO_STREAM: Mutex<Option<OutputEnum>> = Mutex::new(None);
pub struct SegmentInfos<D> {
pub counter: i64,
pub version: i64,
generation: i64,
last_generation: i64,
pub user_data: HashMap<String, String>,
pub(crate) segments: Vec<SegmentCommitInfo<D>>,
pub(crate) dropped_segment_commit_infos: HashMap<Arc<str>, SegmentCommitInfo<D>>,
id: Option<[u8; StringHelper::ID_LENGTH]>,
lucene_version: Option<Version>,
min_segment_lucene_version: Option<Version>,
index_created_version_major: i32,
pub(crate) pending_commit: bool,
}
impl<D> SegmentInfos<D> {
pub fn new(index_created_version_major: i32) -> Result<SegmentInfos<D>> {
if index_created_version_major > LATEST.major {
return Err(LuceneError::illegal_argument(format!(
"indexCreatedVersionMajor is in the future: {index_created_version_major}"
)));
}
if index_created_version_major < 6 {
return Err(LuceneError::illegal_argument(format!(
"indexCreatedVersionMajor must be >= 6, got: {index_created_version_major}"
)));
}
Ok(SegmentInfos {
counter: 0,
version: 0,
generation: 0,
last_generation: 0,
user_data: HashMap::new(),
segments: Vec::new(),
dropped_segment_commit_infos: HashMap::new(),
id: None,
lucene_version: None,
min_segment_lucene_version: None,
index_created_version_major,
pending_commit: false,
})
}
pub fn index_of(&self, seg_id: &str) -> Option<&SegmentCommitInfo<D>> {
self
.segments
.iter()
.find(|sci| sci.info.get_id_key() == seg_id)
.or_else(|| self.dropped_segment_commit_infos.get(seg_id))
}
pub(crate) fn index_of_live(&self, seg_id: &str) -> Option<&SegmentCommitInfo<D>> {
self
.segments
.iter()
.find(|sci| sci.info.get_id_key() == seg_id)
}
pub fn info(&self, i: usize) -> Option<&SegmentCommitInfo<D>> {
self.segments.get(i)
}
#[cfg(test)]
pub fn info_idx_mut(&mut self, i: usize) -> Option<&mut SegmentCommitInfo<D>> {
self.segments.get_mut(i)
}
pub fn index_of_mut(&mut self, seg_id: &str) -> Option<&mut SegmentCommitInfo<D>> {
if let Some(index) = self
.segments
.iter()
.position(|sci| sci.info.get_id_key() == seg_id)
{
return self.segments.get_mut(index);
}
self.dropped_segment_commit_infos.get_mut(seg_id)
}
pub fn get_segments_file_name(&self) -> Option<String> {
IndexFileNames::file_name_from_generation(IndexFileNames::SEGMENTS, "", self.last_generation)
}
pub fn get_next_pending_generation(&self) -> i64 {
if self.generation == -1 {
1
} else {
self.generation + 1
}
}
pub fn get_id(&self) -> Option<&[u8; StringHelper::ID_LENGTH]> {
self.id.as_ref()
}
pub fn read_commit(directory: Arc<D>, segment_file_name: &str) -> Result<SegmentInfos<D>>
where
D: Directory,
{
Self::read_commit_with_file_min_version(directory, segment_file_name, *MIN_SUPPORTED_MAJOR)
}
pub fn read_commit_with_file_min_version(
directory: Arc<D>,
segment_file_name: &str,
min_supported_major_version: i32,
) -> Result<SegmentInfos<D>>
where
D: Directory,
{
let generation = generation_from_segments_file_name(segment_file_name)?;
let mut input = directory.open_checksum_input(segment_file_name)?;
let read_result =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(
|| match SegmentInfos::read_commit_impl(
directory.clone(),
&mut input,
generation,
min_supported_major_version,
) {
Err(error) => {
let is_unexpected_file_read_error = match error.kind() {
LuceneErrorKind::Eof(_) | LuceneErrorKind::NoSuchFile(_) => true,
LuceneErrorKind::Io { source, .. } | LuceneErrorKind::IoWithPath { source, .. } => {
matches!(
source.kind(),
ErrorKind::NotFound | ErrorKind::UnexpectedEof
)
},
_ => false,
};
if is_unexpected_file_read_error {
let mut corrupt_index_error = LuceneError::corrupt_index(format!(
"Unexpected file read error while reading index. (resource={input})"
));
corrupt_index_error.add_suppressed(error);
Err(corrupt_index_error)
} else {
Err(error)
}
},
result => result,
},
));
let close_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| input.close()));
IOUtils::use_or_suppress_caught_result(read_result, close_result)
}
pub fn read_commit_with_input<II>(
directory: Arc<D>,
input: &mut II,
generation: i64,
) -> Result<Self>
where
D: Directory,
II: ChecksumIndexInput,
{
Self::read_commit_impl(directory, input, generation, *MIN_SUPPORTED_MAJOR)
}
pub fn read_commit_impl<II>(
directory: Arc<D>,
input: &mut II,
generation: i64,
min_supported_major_version: i32,
) -> Result<Self>
where
D: Directory,
II: ChecksumIndexInput,
{
let mut format = -1;
let read_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<Self> {
let magic = CodecUtil::read_be_int(input)?;
if magic != CodecUtil::CODEC_MAGIC {
return Err(LuceneError::index_format_too_old_with_version(
input,
magic,
CodecUtil::CODEC_MAGIC,
CodecUtil::CODEC_MAGIC,
));
}
format = CodecUtil::check_header_no_magic(input, "segments", VERSION_74, VERSION_CURRENT)?;
let mut id = [0u8; StringHelper::ID_LENGTH];
let id_len = id.len();
debug_assert!(id_len <= i32::MAX as usize);
input.read_bytes(&mut id, 0, id_len)?;
CodecUtil::check_index_header_suffix(input, base36(generation, &mut [0; 14]))?;
let lucene_version =
Version::from_bits(input.read_vint()?, input.read_vint()?, input.read_vint()?)?;
let index_created_version = input.read_vint()?;
debug_assert!(index_created_version >= 0);
if lucene_version.major < index_created_version {
return Err(LuceneError::corrupt_index(format!(
"Creation version [{index_created_version}] can't be greater than the version that wrote the segment infos: [{lucene_version}]"
)));
}
if index_created_version < min_supported_major_version {
let reason = format!(
"This index was initially created with Lucene {}.x while the current version is {} and Lucene only supports reading {}",
index_created_version,
*LATEST,
if min_supported_major_version == *MIN_SUPPORTED_MAJOR {
"the current and previous major versions".to_string()
} else {
format!("from version {min_supported_major_version} upwards")
}
);
return Err(LuceneError::index_format_too_old(input, reason));
}
let mut infos = Self::new(index_created_version)?;
infos.id = Some(id);
infos.generation = generation;
infos.last_generation = generation;
infos.lucene_version = Some(lucene_version);
Self::parse_segment_infos(directory, input, &mut infos, format)?;
Ok(infos)
}));
if format >= VERSION_74 {
match read_result {
Ok(Ok(infos)) => {
CodecUtil::check_footer_with_error::<(), _>(input, None)?;
Ok(infos)
},
read_result => {
CodecUtil::check_footer_with_error(input, Some(read_result))?;
Err(LuceneError::illegal_state(
"footer check returned after a prior segment infos read failure",
))
},
}
} else {
IOUtils::rethrow_always(read_result)
}
}
pub fn parse_segment_infos<DI>(
directory: Arc<D>,
input: &mut DI,
infos: &mut SegmentInfos<D>,
format: i32,
) -> Result<()>
where
D: Directory,
DI: DataInput,
{
infos.version = CodecUtil::read_be_long(input)?;
let counter_value = input.read_vlong()?;
debug_assert!(counter_value >= 0);
infos.counter = counter_value;
let num_segments = CodecUtil::read_be_int(input)?;
if num_segments < 0 {
return Err(LuceneError::corrupt_index(format!(
"Invalid segment count: {num_segments} (resource={input})"
)));
}
if num_segments > 0 {
infos.min_segment_lucene_version = Some(Version::from_bits(
input.read_vint()?,
input.read_vint()?,
input.read_vint()?,
)?);
}
let mut total_docs = 0i64;
let mut seg_name = String::new();
for _ in 0..num_segments {
match input.read_string()? {
Cow::Borrowed(name) => name.clone_into(&mut seg_name),
Cow::Owned(name) => seg_name = name,
}
let mut segment_id = [0u8; StringHelper::ID_LENGTH];
let segment_id_len = segment_id.len();
debug_assert!(segment_id_len <= i32::MAX as usize);
input.read_bytes(&mut segment_id, 0, segment_id_len)?;
let codec = Self::read_codec(input)?;
let mut info = codec.segment_info_format().read(
directory.clone(),
&seg_name,
&segment_id,
IO_CONTEXT_DEFAULT.as_ref().map_err(Clone::clone)?,
)?;
info.set_codec(codec)?;
let max_doc = info.max_doc()?;
total_docs += i64::from(max_doc);
let del_gen = CodecUtil::read_be_long(input)?;
let del_count = CodecUtil::read_be_int(input)?;
if del_count < 0 || del_count > max_doc {
return Err(LuceneError::corrupt_index(format!(
"Invalid deletion count: {del_count} vs maxDoc={max_doc}, (resource={input})"
)));
}
let field_infos_gen = CodecUtil::read_be_long(input)?;
let dv_gen = CodecUtil::read_be_long(input)?;
let soft_del_count = CodecUtil::read_be_int(input)?;
if soft_del_count < 0 || soft_del_count > max_doc {
return Err(LuceneError::corrupt_index(format!(
"Invalid soft deletion count: {soft_del_count} vs maxDoc={max_doc}, (resource={input})"
)));
}
let combined_del_count = soft_del_count.wrapping_add(del_count);
if combined_del_count > max_doc {
return Err(LuceneError::corrupt_index(format!(
"Invalid combined deletion count: {} vs maxDoc={}, (resource={})",
combined_del_count, max_doc, input
)));
}
let sci_id = if format > VERSION_74 {
match input.read_byte()? {
1 => {
let mut id = [0u8; StringHelper::ID_LENGTH];
let id_len = id.len();
debug_assert!(id_len <= i32::MAX as usize);
input.read_bytes(&mut id, 0, id_len)?;
Some(id)
},
0 => None,
marker => {
return Err(LuceneError::corrupt_index(format!(
"Invalid SegmentCommitInfo ID marker: {marker}"
)));
},
}
} else {
None
};
let info = Arc::new(info);
let mut si_per_commit = SegmentCommitInfo::new(
info,
del_count,
soft_del_count,
del_gen,
field_infos_gen,
dv_gen,
sci_id,
);
si_per_commit.set_owned_field_infos_files(input.read_set_of_strings()?);
let num_dv_fields = CodecUtil::read_be_int(input)?;
let dv_update_files = if num_dv_fields == 0 {
HashMap::new()
} else {
let mut map = HashMap::new();
for _ in 0..num_dv_fields {
map.insert(CodecUtil::read_be_int(input)?, input.read_set_of_strings()?);
}
map
};
si_per_commit.set_doc_values_updates_files(std::borrow::Cow::Owned(dv_update_files));
let segment_index = infos.segments.len();
infos.add(si_per_commit)?;
let info = &infos.segments[segment_index].info;
let segment_version = info.get_version_ref().ok_or_else(|| {
LuceneError::corrupt_index(format!(
"segment {} does not record a Lucene version (resource={})",
info, input
))
})?;
let min_segment_lucene_version =
infos.min_segment_lucene_version.as_ref().ok_or_else(|| {
LuceneError::corrupt_index(format!(
"segments file does not record minSegmentLuceneVersion (resource={input})"
))
})?;
if !segment_version.on_or_after(min_segment_lucene_version) {
return Err(LuceneError::corrupt_index(format!(
"segments file recorded minSegmentLuceneVersion={} but segment={} has older version={} (resource={})",
min_segment_lucene_version, info, segment_version, input
)));
}
if infos.index_created_version_major >= 7
&& segment_version.major < infos.index_created_version_major
{
return Err(LuceneError::corrupt_index(format!(
"segments file recorded indexCreatedVersionMajor={} but segment={} has older version={} (resource={})",
infos.index_created_version_major, info, segment_version, input
)));
}
if infos.index_created_version_major >= 7 && info.get_min_version_ref().is_none() {
return Err(LuceneError::corrupt_index(format!(
"segments infos must record minVersion with indexCreatedVersionMajor={} (resource={})",
infos.index_created_version_major, input
)));
}
}
infos.user_data = input.read_map_of_strings()?;
if total_docs > i64::from(get_actual_max_docs()) {
return Err(LuceneError::corrupt_index(format!(
"Too many documents: an index cannot exceed {} but readers have total maxDoc={}",
get_actual_max_docs(),
total_docs
)));
}
Ok(())
}
pub fn read_codec<DI>(input: &mut DI) -> Result<Codecs>
where
DI: DataInput,
{
let name = input.read_string()?;
match codec::for_name(&name) {
Err(error)
if matches!(error.kind(), LuceneErrorKind::IllegalArgument(_))
&& name.starts_with("Lucene") =>
{
Err(LuceneError::illegal_argument(format!(
"Could not load codec '{name}'. Did you forget to add lucene-backward-codecs.jar?"
)))
},
result => result,
}
}
pub fn read_latest_commit(directory: Arc<D>) -> Result<SegmentInfos<D>>
where
D: Directory,
{
Self::read_latest_commit_with_min_version(directory, *MIN_SUPPORTED_MAJOR)
}
pub fn read_latest_commit_with_min_version(
directory: Arc<D>,
min_supported_major_version: i32,
) -> Result<SegmentInfos<D>>
where
D: Directory,
{
let mut find_segments_file = FindSegmentsFileImpl {
dir: directory,
min_supported_major_version,
};
find_segments_file.run()
}
fn write_with_directory<D2>(&mut self, directory: &D2) -> Result<()>
where
D2: Directory,
{
let next_generation = self.get_next_pending_generation();
let segment_file_name_wrap = IndexFileNames::file_name_from_generation(
IndexFileNames::PENDING_SEGMENTS,
"",
next_generation,
);
let segment_file_name = segment_file_name_wrap.ok_or_else(|| {
LuceneError::illegal_state(format!(
"invalid pending segments generation: {next_generation}"
))
})?;
self.generation = next_generation;
let mut success = false;
let mut segn_output = None;
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<()> {
segn_output = Some(directory.create_output(
&segment_file_name,
IO_CONTEXT_DEFAULT.as_ref().map_err(Clone::clone)?,
)?);
let segn_output = segn_output
.as_mut()
.ok_or_else(|| LuceneError::illegal_state("pending segments output is missing"))?;
self.write(segn_output)?;
segn_output.close()?;
directory.sync(std::slice::from_ref(&segment_file_name))?;
success = true;
Ok(())
}));
if success {
self.pending_commit = true;
} else {
IOUtils::close_while_handling_exception(segn_output.as_mut());
IOUtils::delete_files_ignoring_exceptions(directory, std::iter::once(&segment_file_name));
}
unwrap_caught_result!(result)
}
pub fn write<IO>(&self, out: &mut IO) -> Result<()>
where
IO: IndexOutput,
{
let mut generation_digits = [0; 14];
let v = base36(self.generation, &mut generation_digits);
CodecUtil::write_index_header(
out,
"segments",
VERSION_CURRENT,
&StringHelper::random_id(),
v,
)?;
out.write_vint(LATEST.major)?;
out.write_vint(LATEST.minor)?;
out.write_vint(LATEST.bug_fix)?;
out.write_vint(self.index_created_version_major)?;
CodecUtil::write_be_long(out, self.version)?;
out.write_vlong(self.counter)?;
CodecUtil::write_be_int(out, self.segments.len().try_convert()?)?;
if self.size() > 0 {
let mut min_segment_version: Option<&Version> = None;
for si_per_commit in self.segments.iter() {
let segment_version = si_per_commit.info.version.as_ref().ok_or_else(|| {
LuceneError::illegal_state(format!(
"segment {} does not record a Lucene version",
si_per_commit.info.name
))
})?;
if min_segment_version.is_none_or(|min_version| !segment_version.on_or_after(min_version)) {
min_segment_version = Some(segment_version);
}
}
let min_version = min_segment_version.ok_or_else(|| {
LuceneError::illegal_state("cannot determine minimum segment Lucene version")
})?;
out.write_vint(min_version.major)?;
out.write_vint(min_version.minor)?;
out.write_vint(min_version.bug_fix)?;
}
for si_per_commit in self.segments.iter() {
let si = &si_per_commit.info;
if self.index_created_version_major >= 7 && si.min_version.is_none() {
return Err(LuceneError::illegal_state(format!(
"Segments must record minVersion if they have been created on or after Lucene 7: {}",
si.name
)));
}
out.write_string(&si.name)?;
let segment_id = si.get_id();
let segment_id_len = segment_id.len();
if segment_id_len != StringHelper::ID_LENGTH {
return Err(LuceneError::illegal_state(format!(
"Cannot write segment: invalid id segment={} id={:?}",
si.name, segment_id
)));
}
debug_assert!(segment_id_len <= i32::MAX as usize);
out.write_bytes_with_len(segment_id, segment_id_len)?;
out.write_string(si.get_codec()?.get_name())?;
CodecUtil::write_be_long(out, si_per_commit.get_del_gen())?;
let del_count = si_per_commit.get_del_count();
let max_doc = si.max_doc()?;
if del_count < 0 || del_count > max_doc {
return Err(LuceneError::illegal_state(format!(
"Cannot write segment: invalid maxDoc segment={} maxDoc={} delCount={}",
si.name, max_doc, del_count
)));
}
CodecUtil::write_be_int(out, del_count)?;
CodecUtil::write_be_long(out, si_per_commit.get_field_infos_gen())?;
CodecUtil::write_be_long(out, si_per_commit.get_doc_values_gen())?;
let soft_del_count = si_per_commit.get_soft_del_count();
if soft_del_count < 0 || soft_del_count > max_doc {
return Err(LuceneError::illegal_state(format!(
"Cannot write segment: invalid maxDoc segment={} maxDoc={} softDelCount={}",
si.name, max_doc, soft_del_count
)));
}
CodecUtil::write_be_int(out, soft_del_count)?;
if let Some(sci_id) = si_per_commit.get_id() {
out.write_byte(1)?;
let sci_id_len = sci_id.len();
debug_assert_eq!(
sci_id_len,
StringHelper::ID_LENGTH,
"Invalid SegmentCommitInfo#id: {sci_id:?}"
);
debug_assert!(sci_id_len <= i32::MAX as usize);
out.write_bytes_range(sci_id, 0, sci_id_len)?;
} else {
out.write_byte(0)?;
}
out.write_set_of_strings(si_per_commit.get_field_infos_files())?;
let dv_updates_files = si_per_commit.get_doc_values_updates_files();
let dv_updates_files_len = dv_updates_files.len();
CodecUtil::write_be_int(out, dv_updates_files_len.try_convert()?)?;
for (key, value) in dv_updates_files {
CodecUtil::write_be_int(out, *key)?;
out.write_set_of_strings(value)?;
}
}
out.write_map_of_strings(&self.user_data)?;
CodecUtil::write_footer(out)?;
Ok(())
}
pub fn try_clone(&self) -> Result<Self> {
let mut cloned = Self {
counter: self.counter,
version: self.version,
generation: self.generation,
last_generation: self.last_generation,
user_data: self.user_data.clone(),
segments: Vec::with_capacity(self.segments.len()),
dropped_segment_commit_infos: self.dropped_segment_commit_infos.clone(),
id: self.id,
lucene_version: self.lucene_version.clone(),
min_segment_lucene_version: self.min_segment_lucene_version.clone(),
index_created_version_major: self.index_created_version_major,
pending_commit: self.pending_commit,
};
for segment_commit_info in self.segments.iter() {
cloned.add(segment_commit_info.clone())?;
}
Ok(cloned)
}
pub fn get_version(&self) -> i64 {
self.version
}
pub fn get_generation(&self) -> i64 {
self.generation
}
pub fn get_last_generation(&self) -> i64 {
self.last_generation
}
pub fn update_generation<D1>(&mut self, other: &SegmentInfos<D1>) {
self.last_generation = other.last_generation;
self.generation = other.generation;
}
pub fn update_generation_version_and_counter<D1>(&mut self, other: &SegmentInfos<D1>) {
self.update_generation(other);
self.version = other.version;
self.counter = other.counter;
}
pub fn set_next_write_generation(&mut self, generation: i64) -> Result<()> {
if generation < self.generation {
return Err(LuceneError::illegal_state(format!(
"Cannot decrease generation to {} from current generation {}",
generation, self.generation
)));
}
self.generation = generation;
Ok(())
}
pub fn rollback_commit<D2>(&mut self, directory: &D2)
where
D2: Directory,
{
if self.pending_commit {
self.pending_commit = false;
if let Some(pending) = IndexFileNames::file_name_from_generation(
IndexFileNames::PENDING_SEGMENTS,
"",
self.generation,
) {
IOUtils::delete_files_ignoring_exceptions(directory, std::iter::once(&pending));
}
}
}
pub fn prepare_commit<D2>(&mut self, directory: &D2) -> Result<()>
where
D2: Directory,
{
#[cfg(test)]
let _execution_scope =
ExecutionScope::enter(ExecutionOwner::SegmentInfos, ExecutionMethod::PrepareCommit);
if self.pending_commit {
return Err(LuceneError::illegal_state(
"prepare_commit was already called",
));
}
directory.sync_metadata()?;
self.write_with_directory(directory)?;
Ok(())
}
pub fn files(&self, include_segments_file: bool) -> Result<HashSet<String>> {
let size = self.size();
if size == 1 && !include_segments_file {
return self
.info(0)
.ok_or_else(|| LuceneError::illegal_state("segment was None"))?
.files();
}
let mut files = HashSet::with_capacity(size + usize::from(include_segments_file));
if include_segments_file && let Some(segment_file_name) = self.get_segments_file_name() {
files.insert(segment_file_name);
}
for i in 0..size {
let segment_commit_info = self
.info(i)
.ok_or_else(|| LuceneError::illegal_state("segment was None"))?;
segment_commit_info.append_files_to(&mut files)?;
}
Ok(files)
}
pub(crate) fn files_for_read(
&self,
include_segments_file: bool,
) -> Result<Cow<'_, HashSet<String>>> {
if !include_segments_file && let [info] = self.segments.as_slice() {
info.files_for_read()
} else {
self.files(include_segments_file).map(Cow::Owned)
}
}
pub(crate) fn files_to_sync(&self) -> Result<Vec<String>> {
Ok(match self.files_for_read(false)? {
Cow::Borrowed(files) => files.iter().cloned().collect(),
Cow::Owned(files) => files.into_iter().collect(),
})
}
pub fn finish_commit<D2>(&mut self, directory: &D2) -> Result<String>
where
D2: Directory,
{
#[cfg(test)]
let _execution_scope =
ExecutionScope::enter(ExecutionOwner::SegmentInfos, ExecutionMethod::FinishCommit);
if !self.pending_commit {
return Err(LuceneError::illegal_state("prepare_commit was not called"));
}
let mut success_rename_and_sync = false;
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<String> {
let src = IndexFileNames::file_name_from_generation(
IndexFileNames::PENDING_SEGMENTS,
"",
self.generation,
)
.ok_or_else(|| LuceneError::illegal_state("Failed to generate source file name."))?;
let dest =
IndexFileNames::file_name_from_generation(IndexFileNames::SEGMENTS, "", self.generation)
.ok_or_else(|| LuceneError::illegal_state("Failed to generate destination file name."))?;
directory.rename(&src, &dest)?;
let sync_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<()> {
directory.sync_metadata()?;
success_rename_and_sync = true;
Ok(())
}));
if !success_rename_and_sync {
IOUtils::delete_files_ignoring_exceptions(directory, std::iter::once(&dest));
}
unwrap_caught_result!(sync_result)?;
Ok(dest)
}));
if !success_rename_and_sync {
self.rollback_commit(directory);
}
let dest = unwrap_caught_result!(result)?;
self.pending_commit = false;
self.last_generation = self.generation;
Ok(dest)
}
pub fn commit<D2>(&mut self, dir: &D2) -> Result<()>
where
D2: Directory,
{
self.prepare_commit(dir)?;
self.finish_commit(dir)?;
Ok(())
}
pub fn get_user_data(&self) -> &HashMap<String, String> {
&self.user_data
}
pub fn set_user_data(
&mut self,
data: Option<HashMap<String, String>>,
do_increment_version: bool,
) {
if let Some(new_data) = data {
self.user_data = new_data;
} else {
self.user_data = HashMap::new();
}
if do_increment_version {
self.changed();
}
}
pub fn replace(&mut self, other: Self) -> Result<()> {
self.rollback_segment_infos(other.segments)?;
self.last_generation = other.last_generation;
self.user_data = other.user_data;
Ok(())
}
pub fn total_max_doc(&self) -> Result<i32> {
let mut count = 0i64;
for segment_commit_info in self.segments.iter() {
count += i64::from(segment_commit_info.info.max_doc()?);
}
debug_assert!(count <= i64::from(get_actual_max_docs()));
count.try_convert()
}
pub fn changed(&mut self) {
self.version += 1;
}
pub fn set_version(&mut self, new_version: i64) -> Result<()> {
if new_version < self.version {
return Err(LuceneError::illegal_argument(format!(
"newVersion (={}) cannot be less than current version (={})",
new_version, self.version
)));
}
self.version = new_version;
Ok(())
}
pub(crate) fn apply_merge_changes(
&mut self,
source_ids: &[Arc<str>],
merge_info: &mut Option<SegmentCommitInfo<D>>,
drop_segment: bool,
) -> Result<()> {
if self.index_created_version_major >= 7
&& merge_info
.as_ref()
.and_then(|sci| sci.info.get_min_version())
.is_none()
{
return Err(LuceneError::illegal_argument(
"All segments must record the minVersion for indices created on or after Lucene 7",
));
}
let merged_away = (source_ids.len() > 4).then(|| {
source_ids
.iter()
.map(AsRef::as_ref)
.collect::<HashSet<&str>>()
});
let mut kept = 0;
let mut insertion_index = None;
let removed = self.segments.extract_if(.., |info| {
let info_id = info.info.get_id_key();
let is_source = match &merged_away {
Some(ids) => ids.contains(info_id),
None => source_ids.iter().any(|id| id.as_ref() == info_id),
};
if is_source {
insertion_index.get_or_insert(kept);
} else {
kept += 1;
}
is_source
});
let mut output = None;
for info in removed {
self
.dropped_segment_commit_infos
.insert(Arc::clone(&info.info.id_key), info);
if !drop_segment && output.is_none() {
output = merge_info.take();
if output.is_none() {
break;
}
}
}
if !drop_segment {
let Some(output) = output.or_else(|| merge_info.take()) else {
self.segments.clear();
return Err(LuceneError::illegal_state(
"merge output segment info is missing",
));
};
self.segments.insert(insertion_index.unwrap_or(0), output);
}
Ok(())
}
pub fn create_backup_segment_infos(&self) -> Result<Vec<SegmentCommitInfo<D>>> {
let mut list = Vec::with_capacity(self.segments.len());
for segment_commit_info in self.segments.iter() {
list.push(segment_commit_info.clone());
}
Ok(list)
}
pub fn rollback_segment_infos(&mut self, mut infos: Vec<SegmentCommitInfo<D>>) -> Result<()> {
self.clear();
if self.index_created_version_major >= 7
&& let Some(index) = infos
.iter()
.position(|info| info.info.min_version.is_none())
{
let error = Self::missing_min_version(&infos[index]);
infos.truncate(index);
self.segments = infos;
return Err(error);
}
self.segments = infos;
Ok(())
}
pub fn iter_mut(&mut self) -> &mut [SegmentCommitInfo<D>] {
self.segments.as_mut_slice()
}
pub fn iter(&self) -> &[SegmentCommitInfo<D>] {
self.segments.as_slice()
}
pub fn size(&self) -> usize {
self.segments.len()
}
pub fn add(&mut self, si: SegmentCommitInfo<D>) -> Result<()> {
if self.index_created_version_major >= 7 && si.info.min_version.is_none() {
return Err(Self::missing_min_version(&si));
}
let _id = si.info.get_id_key();
self.segments.push(si);
Ok(())
}
fn missing_min_version(info: &SegmentCommitInfo<D>) -> LuceneError {
LuceneError::illegal_argument(format!(
"All segments must record the minVersion for indices created on or after Lucene 7, but minVersion is missing for segment: {info}"
))
}
pub fn add_all<I>(&mut self, sis: I) -> Result<()>
where
I: IntoIterator<Item = SegmentCommitInfo<D>>,
{
let sis = sis.into_iter();
self.segments.reserve(sis.size_hint().0);
for si in sis {
self.add(si)?;
}
Ok(())
}
pub fn clear(&mut self) {
self
.dropped_segment_commit_infos
.reserve(self.segments.len());
for info in self.segments.drain(..) {
self
.dropped_segment_commit_infos
.insert(Arc::clone(&info.info.id_key), info);
}
}
pub fn remove_with_id(&mut self, si_id: &str) -> bool {
let Some(idx) = self
.segments
.iter()
.position(|sci| sci.info.get_id_key() == si_id)
else {
return false;
};
let info = self.segments.remove(idx);
self
.dropped_segment_commit_infos
.insert(Arc::clone(&info.info.id_key), info);
true
}
pub fn remove(&mut self, index: usize) {
if index >= self.segments.len() {
return;
}
let info = self.segments.remove(index);
self
.dropped_segment_commit_infos
.insert(Arc::clone(&info.info.id_key), info);
}
pub(crate) fn remove_dropped_segment_commit_info(
&mut self,
si_id: &str,
) -> Option<SegmentCommitInfo<D>> {
self.dropped_segment_commit_infos.remove(si_id)
}
pub fn contains(&self, si_id: &str) -> bool {
self.index_of_live(si_id).is_some()
}
pub fn get_commit_lucene_version(&self) -> Option<&Version> {
self.lucene_version.as_ref()
}
pub fn get_min_segment_lucene_version(&self) -> Option<&Version> {
self.min_segment_lucene_version.as_ref()
}
pub fn get_index_created_version_major(&self) -> i32 {
self.index_created_version_major
}
pub fn seg_ids(&self) -> Vec<String> {
self
.segments
.iter()
.map(|s| s.info.get_id_key().to_string())
.collect()
}
}
impl<D> fmt::Display for SegmentInfos<D> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{}: ",
self
.get_segments_file_name()
.unwrap_or_else(|| "null".to_string())
)?;
for (i, segment_commit_info) in self.segments.iter().enumerate() {
if i > 0 {
write!(f, " ")?;
}
write!(f, "{segment_commit_info}")?
}
Ok(())
}
}
pub trait FindSegmentsFile {
type V;
type D: Directory;
fn get_directory_point(&self) -> Arc<Self::D>;
fn run_with_commit<T>(&mut self, commit: &T) -> Result<Self::V>
where
T: IndexCommit<Directory = Arc<Self::D>>,
{
if !self
.get_directory_point()
.is_same_identity(&*commit.get_directory())
{
return Err(LuceneError::io(Error::other(
"the specified commit does not match the specified Directory",
)));
}
self.do_body(commit.get_segments_file_name())
}
fn run(&mut self) -> Result<Self::V> {
let mut last_gen: i64;
let mut gen_: i64 = -1;
let mut exc: Option<LuceneError> = None;
let directory = self.get_directory_point();
loop {
last_gen = gen_;
let mut files = directory.list_all()?;
let mut files2 = directory.list_all()?;
files.sort_unstable();
files2.sort_unstable();
if files != files2 {
continue;
}
gen_ = get_last_commit_generation(&files)?;
if get_info_stream().is_some() {
message(&format!("directory listing gen={gen_}"))?;
}
if gen_ == -1 {
return Err(LuceneError::index_not_found(format!(
"No segments* file found in the {}: files: {:?}",
directory, files
)));
} else if gen_ > last_gen {
let segment_file_name =
IndexFileNames::file_name_from_generation(IndexFileNames::SEGMENTS, "", gen_)
.ok_or_else(|| LuceneError::illegal_state("Failed to generate segment file name."))?;
match self.do_body(&segment_file_name) {
Ok(result) => {
if get_info_stream().is_some() {
message(&format!("success on {segment_file_name}"))?;
}
return Ok(result);
},
Err(err) if err.is_io_error() => {
let error_message = err.to_string();
if exc.is_none() {
exc = Some(err);
}
if get_info_stream().is_some() {
message(&format!(
"primary error on '{}': {}; will retry: gen = {}",
segment_file_name, error_message, gen_
))?;
}
},
Err(err) => return Err(err),
}
} else {
return Err(exc.unwrap_or_else(|| {
LuceneError::illegal_state("Unexpected error during FindSegmentsFile::run")
}));
}
}
}
fn do_body(&mut self, segment_file_name: &str) -> Result<Self::V>;
}
pub struct FindSegmentsFileImpl<D> {
pub(crate) dir: Arc<D>,
pub(crate) min_supported_major_version: i32,
}
impl<D> FindSegmentsFile for FindSegmentsFileImpl<D>
where
D: Directory,
{
type V = SegmentInfos<D>;
type D = D;
fn get_directory_point(&self) -> Arc<Self::D> {
self.dir.clone()
}
fn do_body(&mut self, segment_file_name: &str) -> Result<Self::V> {
SegmentInfos::read_commit_with_file_min_version(
self.dir.clone(),
segment_file_name,
self.min_supported_major_version,
)
}
}
pub const VERSION_74: i32 = 9;
pub const VERSION_86: i32 = 10;
pub(crate) const VERSION_CURRENT: i32 = VERSION_86;
pub(crate) const OLD_SEGMENTS_GEN: &str = "segments.gen";
pub fn set_info_stream<T>(output: T) -> Result<()>
where
T: Into<Option<OutputEnum>>,
{
let mut info_stream = INFO_STREAM.lock();
*info_stream = output.into();
Ok(())
}
pub fn get_info_stream() -> parking_lot::MutexGuard<'static, Option<OutputEnum>> {
INFO_STREAM.lock()
}
pub fn message(msg: &str) -> Result<()> {
let mut info_stream = INFO_STREAM.lock();
if let Some(stream) = info_stream.as_mut() {
let current_thread = std::thread::current();
let thread_name = current_thread.name().unwrap_or("unnamed");
let _ = writeln!(stream, "SIS [{thread_name}]: {msg}");
}
Ok(())
}
pub fn get_last_commit_generation(files: &[String]) -> Result<i64> {
let mut max = -1;
for file in files {
if file.starts_with(IndexFileNames::SEGMENTS)
&& !file.starts_with(OLD_SEGMENTS_GEN)
{
let gen_ = generation_from_segments_file_name(file)?;
if gen_ > max {
max = gen_;
}
}
}
Ok(max)
}
pub fn get_last_commit_generation_from_directory<D>(directory: &D) -> Result<i64>
where
D: Directory,
{
let files = directory.list_all()?;
get_last_commit_generation(&files)
}
pub fn get_last_commit_segments_file_name(files: &[String]) -> Result<Option<String>> {
let last_gen = get_last_commit_generation(files)?;
Ok(IndexFileNames::file_name_from_generation(
IndexFileNames::SEGMENTS,
"",
last_gen,
))
}
pub fn get_last_commit_segments_file_name_from_directory<D>(directory: &D) -> Result<Option<String>>
where
D: Directory,
{
let last_gen = get_last_commit_generation_from_directory(directory)?;
Ok(IndexFileNames::file_name_from_generation(
IndexFileNames::SEGMENTS,
"",
last_gen,
))
}
pub fn generation_from_segments_file_name(file_name: &str) -> Result<i64> {
if file_name == OLD_SEGMENTS_GEN {
Err(LuceneError::illegal_argument(format!(
"\"{}\" is not a valid segment file name since 4.0",
OLD_SEGMENTS_GEN
)))
} else if file_name == IndexFileNames::SEGMENTS {
Ok(0)
} else if file_name.starts_with(IndexFileNames::SEGMENTS) {
let generation_str = &file_name[IndexFileNames::SEGMENTS.len() + 1..];
match i64::from_str_radix(generation_str, 36) {
Ok(generation) => Ok(generation),
Err(_) => Err(LuceneError::number_format(format!(
"Failed to parse generation from file name: \"{file_name}\""
))),
}
} else {
Err(LuceneError::illegal_argument(format!(
"fileName \"{file_name}\" is not a segments file"
)))
}
}