use crate::fold::{Fold, FoldCfg, Loc};
use crate::part::Part;
use crate::types::{ContentOp, PieceHash, Record};
use anyhow::{bail, Result};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
#[derive(Clone, Copy, Debug, Default)]
pub struct RefoldStats {
pub parts_in: usize,
pub parts_out: usize,
pub records_kept: usize,
pub records_dropped: usize,
pub tombstones_dropped: usize,
pub pieces_kept: usize,
pub pieces_dropped: usize,
pub fold_bytes_before: u64,
pub fold_bytes_after: u64,
pub stale_generation_left: bool,
}
impl RefoldStats {
pub fn bytes_reclaimed(&self) -> u64 {
self.fold_bytes_before.saturating_sub(self.fold_bytes_after)
}
}
#[allow(clippy::type_complexity)]
fn plan_refold(
parts: &[Arc<Part>],
control: &crate::control::OperationControl,
) -> Result<(Vec<Vec<usize>>, Vec<(Loc, PieceHash)>, RefoldStats)> {
let mut st = RefoldStats { parts_in: parts.len(), ..Default::default() };
let visible = super::read::visibility(parts)?;
let live = visible.rows;
st.records_dropped = visible.superseded + visible.tombstones;
st.tombstones_dropped = visible.tombstones;
st.records_kept = live.iter().map(|v| v.len()).sum();
let mut wanted: HashMap<PieceHash, Loc> = HashMap::new();
for (pi, rows) in live.iter().enumerate() {
for &row in rows {
control.check("content refold")?;
for content in parts[pi].record(row)?.contents {
for op in content.ops {
let ContentOp::Piece { hash, .. } = op else { continue };
if wanted.contains_key(&hash) {
continue;
}
let loc = parts[pi].find_piece(&hash)?.ok_or_else(|| {
anyhow::anyhow!(
"a record resolved by current authority references piece {hash}, which no part locates"
)
})?;
wanted.insert(hash, loc);
}
}
}
}
let total_pieces: usize = {
let mut all = HashSet::new();
for p in parts {
for i in 0..p.piece_count()? {
control.check("content refold")?;
all.insert(p.piece(i)?.1);
}
}
all.len()
};
st.pieces_kept = wanted.len();
st.pieces_dropped = total_pieces.saturating_sub(wanted.len());
let mut order: Vec<(Loc, PieceHash)> = wanted.iter().map(|(h, l)| (*l, *h)).collect();
order.sort_by_key(|(l, _)| (l.block_id, l.in_off));
Ok((live, order, st))
}
pub(crate) type RefoldedMemberPart = (String, u64, u64, u32, String);
#[allow(clippy::too_many_arguments, clippy::type_complexity)]
pub(crate) fn refold_into_container_with_control_and_limits(
container: std::sync::Arc<std::sync::Mutex<crate::container::Container>>,
parts: &[Arc<Part>],
seqs: &[(u64, u64)],
old_fold: &Fold,
old_gen: u32,
cfg: FoldCfg,
control: &crate::control::OperationControl,
read_limits: crate::read_limits::ReadLimits,
) -> Result<(u32, Vec<RefoldedMemberPart>, RefoldStats, Fold)> {
let build = (|| -> Result<(u32, Vec<RefoldedMemberPart>, RefoldStats, Fold)> {
control.check("content refold")?;
if parts.len() != seqs.len() {
bail!("every part needs its committed sequence range");
}
let new_gen = old_gen
.checked_add(1)
.filter(|generation| *generation <= crate::fold::MAX_FOLD_GENERATION)
.ok_or_else(|| anyhow::anyhow!("fold generation namespace exhausted"))?;
let (live, order, mut st) = plan_refold(parts, control)?;
st.fold_bytes_before = old_fold.disk_bytes();
let mut nf =
Fold::open_container_writer(container.clone(), new_gen, cfg, None, &[], read_limits)?;
let mut remap: HashMap<PieceHash, Loc> = HashMap::with_capacity(order.len());
for (loc, hash) in &order {
control.check("content refold")?;
let bytes = old_fold.read_verified(*loc, *hash)?;
let put = nf.put_hashed(&bytes, *hash)?;
remap.insert(*hash, put.loc);
}
control.check("content refold")?;
nf.sync()?;
st.fold_bytes_after = nf.disk_bytes();
let mut out = Vec::new();
let last_surviving = live.iter().rposition(|rows| !rows.is_empty());
let mut pending_lo = seqs.first().expect("refold has at least one input part").0;
for (pi, rows) in live.iter().enumerate() {
control.check("content refold")?;
if rows.is_empty() {
continue; }
let recs: Vec<Record> =
rows.iter().map(|&r| parts[pi].record(r)).collect::<Result<_>>()?;
let lo = pending_lo;
let hi = if Some(pi) == last_surviving {
seqs.last().expect("parts and sequence ranges are parallel").1
} else {
seqs[pi].1
};
pending_lo = hi.saturating_add(1);
let file = format!("part-r{new_gen:04}-{lo:08}-{hi:08}.part");
let member = container.lock().expect("container lock poisoned").begin_member(&file)?;
let built = crate::part::build_full_into(
member,
&recs,
&[],
lo,
hi,
cfg.level,
|h| remap.get(h).copied(),
&HashMap::new(),
read_limits,
);
let (meta, member) = match built {
Ok(v) => v,
Err(error) => {
container.lock().expect("container lock poisoned").abandon_open_member();
return Err(error);
}
};
let digest = {
let mut c = container.lock().expect("container lock poisoned");
PieceHash(c.finish_member(member)?).to_hex()
};
out.push((file, lo, hi, meta.n_records, digest));
}
if out.is_empty() {
let lo = seqs.first().expect("refold has at least one input part").0;
let hi = seqs.last().expect("refold has at least one input part").1;
let file = format!("part-r{new_gen:04}-{lo:08}-{hi:08}.part");
let member = container.lock().expect("container lock poisoned").begin_member(&file)?;
let built = crate::part::build_full_into(
member,
&[],
&[],
lo,
hi,
cfg.level,
|_| None,
&HashMap::new(),
read_limits,
);
let (meta, member) = match built {
Ok(value) => value,
Err(error) => {
container.lock().expect("container lock poisoned").abandon_open_member();
return Err(error);
}
};
let digest = {
let mut container = container.lock().expect("container lock poisoned");
PieceHash(container.finish_member(member)?).to_hex()
};
out.push((file, lo, hi, meta.n_records, digest));
}
control.check("content refold")?;
st.parts_out = out.len();
Ok((new_gen, out, st, nf))
})();
match build {
Ok(v) => Ok(v),
Err(error) => {
let _ = container.lock().expect("container lock poisoned").discard_staged();
Err(error)
}
}
}