use std::fs;
use std::{io, time};
use crate::grin_core::core::hash::{Hash, Hashed};
use crate::grin_core::core::pmmr::{self, family, Backend};
use crate::grin_core::core::BlockHeader;
use crate::grin_core::ser::{PMMRable, ProtocolVersion};
use crate::leaf_set::LeafSet;
use crate::prune_list::PruneList;
use crate::types::{AppendOnlyFile, DataFile, SizeEntry, SizeInfo};
use croaring::Bitmap;
use std::convert::TryInto;
use std::path::{Path, PathBuf};
const PMMR_HASH_FILE: &str = "pmmr_hash.bin";
const PMMR_DATA_FILE: &str = "pmmr_data.bin";
const PMMR_LEAF_FILE: &str = "pmmr_leaf.bin";
const PMMR_PRUN_FILE: &str = "pmmr_prun.bin";
const PMMR_SIZE_FILE: &str = "pmmr_size.bin";
const REWIND_FILE_CLEANUP_DURATION_SECONDS: u64 = 60 * 60 * 24; pub const PMMR_FILES: [&str; 4] = [
PMMR_HASH_FILE,
PMMR_DATA_FILE,
PMMR_LEAF_FILE,
PMMR_PRUN_FILE,
];
pub struct PMMRBackend<T: PMMRable> {
data_dir: PathBuf,
prunable: bool,
hash_file: DataFile<Hash>,
data_file: DataFile<T::E>,
leaf_set: LeafSet,
prune_list: PruneList,
}
impl<T: PMMRable> Backend<T> for PMMRBackend<T> {
fn append(&mut self, data: &T, hashes: &[Hash]) -> Result<(), String> {
let size = self
.data_file
.append(&data.as_elmt())
.map_err(|e| format!("Failed to append data to file. {}", e))?;
self.hash_file
.extend_from_slice(hashes)
.map_err(|e| format!("Failed to append hash to file. {}", e))?;
if self.prunable {
let pos =
pmmr::insertion_to_pmmr_index(size + self.prune_list.get_total_leaf_shift() - 1);
self.leaf_set.add(pos);
}
Ok(())
}
fn append_pruned_subtree(&mut self, hash: Hash, pos0: u64) -> Result<(), String> {
if !self.prunable {
return Err("Not prunable, cannot append pruned subtree.".into());
}
self.hash_file
.append(&hash)
.map_err(|e| format!("Failed to append subtree hash to file. {}", e))?;
self.prune_list.append(pos0);
Ok(())
}
fn append_hash(&mut self, hash: Hash) -> Result<(), String> {
self.hash_file
.append(&hash)
.map_err(|e| format!("Failed to append hash to file. {}", e))?;
Ok(())
}
fn get_from_file(&self, pos0: u64) -> Option<Hash> {
if self.is_compacted(pos0) {
return None;
}
let shift = self.prune_list.get_shift(pos0);
self.hash_file.read(1 + pos0 - shift)
}
fn get_peak_from_file(&self, pos0: u64) -> Option<Hash> {
let shift = self.prune_list.get_shift(pos0);
self.hash_file.read(1 + pos0 - shift)
}
fn get_data_from_file(&self, pos0: u64) -> Option<T::E> {
if !pmmr::is_leaf(pos0) {
return None;
}
if self.is_compacted(pos0) {
return None;
}
let flatfile_pos = pmmr::n_leaves(pos0 + 1);
let shift = self.prune_list.get_leaf_shift(1 + pos0);
self.data_file.read(flatfile_pos - shift)
}
fn get_hash(&self, pos0: u64) -> Option<Hash> {
if self.prunable && pmmr::is_leaf(pos0) && !self.leaf_set.includes(pos0) {
return None;
}
self.get_from_file(pos0)
}
fn get_data(&self, pos0: u64) -> Option<T::E> {
if !pmmr::is_leaf(pos0) {
return None;
}
if self.prunable && !self.leaf_set.includes(pos0) {
return None;
}
self.get_data_from_file(pos0)
}
fn remove_from_leaf_set(&mut self, pos0: u64) {
self.leaf_set.remove(pos0);
}
fn leaf_pos_iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
if self.prunable {
Box::new(self.leaf_set.iter().map(|x| x - 1))
} else {
panic!("leaf_pos_iter not implemented for non-prunable PMMR")
}
}
fn n_unpruned_leaves(&self) -> u64 {
if self.prunable {
self.leaf_set.len() as u64
} else {
pmmr::n_leaves(self.unpruned_size())
}
}
fn n_unpruned_leaves_to_index(&self, to_index: u64) -> u64 {
if self.prunable {
self.leaf_set.n_unpruned_leaves_to_index(to_index)
} else {
pmmr::n_leaves(pmmr::insertion_to_pmmr_index(to_index))
}
}
fn leaf_idx_iter(&self, from_idx: u64) -> Box<dyn Iterator<Item = u64> + '_> {
let from_pos = 1 + pmmr::insertion_to_pmmr_index(from_idx);
if self.prunable {
Box::new(
self.leaf_set
.iter()
.skip_while(move |x| *x < from_pos)
.map(|x| pmmr::n_leaves(x).saturating_sub(1)),
)
} else {
panic!("leaf_idx_iter not implemented for non-prunable PMMR")
}
}
fn rewind(&mut self, position: u64, rewind_rm_pos: &Bitmap) -> Result<(), String> {
if self.prunable {
self.leaf_set.rewind(position, rewind_rm_pos);
}
let shift = if position == 0 {
0
} else {
self.prune_list.get_shift(position - 1)
};
self.hash_file.rewind(position - shift);
let flatfile_pos = pmmr::n_leaves(position);
let leaf_shift = if position == 0 {
0
} else {
self.prune_list.get_leaf_shift(position)
};
self.data_file.rewind(flatfile_pos - leaf_shift);
Ok(())
}
fn reset_prune_list(&mut self) {
let bitmap = Bitmap::new();
self.prune_list = PruneList::new(Some(self.data_dir.join(PMMR_PRUN_FILE)), bitmap);
if let Err(e) = self.prune_list.flush() {
error!("Flushing reset prune list: {}", e);
}
}
fn remove(&mut self, pos0: u64) -> Result<(), String> {
assert!(self.prunable, "Remove on non-prunable MMR");
self.leaf_set.remove(pos0);
Ok(())
}
fn release_files(&mut self) {
self.data_file.release();
self.hash_file.release();
}
fn snapshot(&self, header: &BlockHeader) -> Result<(), String> {
self.leaf_set
.snapshot(header)
.map_err(|_| format!("Failed to save copy of leaf_set for {}", header.hash()))?;
Ok(())
}
fn dump_stats(&self) {
debug!(
"pmmr backend: unpruned: {}, hashes: {}, data: {}, leaf_set: {}, prune_list: {}",
self.unpruned_size(),
self.hash_size(),
self.data_size(),
self.leaf_set.len(),
self.prune_list.len(),
);
}
}
impl<T: PMMRable> PMMRBackend<T> {
pub fn new<P: AsRef<Path>>(
data_dir: P,
prunable: bool,
version: ProtocolVersion,
header: Option<&BlockHeader>,
) -> io::Result<PMMRBackend<T>> {
let data_dir = data_dir.as_ref();
let size_info = if let Some(fixed_size) = T::elmt_size() {
SizeInfo::FixedSize(fixed_size)
} else {
SizeInfo::VariableSize(Box::new(AppendOnlyFile::open(
data_dir.join(PMMR_SIZE_FILE),
SizeInfo::FixedSize(SizeEntry::LEN as u16),
version,
)?))
};
let hash_size_info = SizeInfo::FixedSize(Hash::LEN.try_into().unwrap());
let hash_file = DataFile::open(&data_dir.join(PMMR_HASH_FILE), hash_size_info, version)?;
let data_file = DataFile::open(&data_dir.join(PMMR_DATA_FILE), size_info, version)?;
let leaf_set_path = data_dir.join(PMMR_LEAF_FILE);
if let Some(header) = header {
let leaf_snapshot_path = format!(
"{}.{}",
data_dir.join(PMMR_LEAF_FILE).to_str().unwrap(),
header.hash()
);
LeafSet::copy_snapshot(&leaf_set_path, &PathBuf::from(leaf_snapshot_path))?;
}
let leaf_set = LeafSet::open(&leaf_set_path)?;
let prune_list = PruneList::open(&data_dir.join(PMMR_PRUN_FILE))?;
Ok(PMMRBackend {
data_dir: data_dir.to_path_buf(),
prunable,
hash_file,
data_file,
leaf_set,
prune_list,
})
}
fn is_pruned(&self, pos0: u64) -> bool {
self.prune_list.is_pruned(pos0)
}
fn is_pruned_root(&self, pos0: u64) -> bool {
self.prune_list.is_pruned_root(pos0)
}
fn is_compacted(&self, pos0: u64) -> bool {
if self.leaf_set.includes(pos0) {
return false;
}
!self.is_pruned_root(pos0) && self.is_pruned(pos0)
}
pub fn unpruned_size(&self) -> u64 {
self.hash_size() + self.prune_list.get_total_shift()
}
pub fn data_size(&self) -> u64 {
self.data_file.size()
}
pub fn hash_size(&self) -> u64 {
self.hash_file.size()
}
pub fn sync(&mut self) -> io::Result<()> {
Ok(())
.and(self.hash_file.flush())
.and(self.data_file.flush())
.and(self.sync_leaf_set())
.and(self.prune_list.flush())
.map_err(|e| {
io::Error::new(
io::ErrorKind::Interrupted,
format!("Could not sync pmmr to disk: {:?}", e),
)
})
}
fn sync_leaf_set(&mut self) -> io::Result<()> {
if !self.prunable {
return Ok(());
}
self.leaf_set.flush()
}
pub fn discard(&mut self) {
self.hash_file.discard();
self.data_file.discard();
self.leaf_set.discard();
}
pub fn check_compact(&mut self, cutoff_pos: u64, rewind_rm_pos: &Bitmap) -> io::Result<bool> {
assert!(self.prunable, "Trying to compact a non-prunable PMMR");
let (leaves_removed, pos_to_rm) = self.pos_to_rm(cutoff_pos, rewind_rm_pos);
{
let pos_to_rm = map_vec!(pos_to_rm, |pos1| {
let shift = self.prune_list.get_shift(pos1 as u64 - 1);
pos1 as u64 - shift
});
self.hash_file.write_tmp_pruned(&pos_to_rm)?;
}
{
let leaf_pos_to_rm = pos_to_rm
.iter()
.map(|x| x as u64)
.filter(|x| pmmr::is_leaf(x - 1))
.collect::<Vec<_>>();
let pos_to_rm = map_vec!(leaf_pos_to_rm, |&pos| {
let flat_pos = pmmr::n_leaves(pos);
let shift = self.prune_list.get_leaf_shift(pos);
flat_pos - shift
});
self.data_file.write_tmp_pruned(&pos_to_rm)?;
}
{
debug!("compact: about to replace hash and data files and rebuild...");
self.hash_file.replace_with_tmp()?;
self.data_file.replace_with_tmp()?;
debug!("compact: ...finished replacing and rebuilding");
}
{
let mut bitmap = self.prune_list.bitmap();
bitmap.or_inplace(&leaves_removed);
self.prune_list = PruneList::new(Some(self.data_dir.join(PMMR_PRUN_FILE)), bitmap);
self.prune_list.flush()?;
}
self.leaf_set.flush()?;
self.clean_rewind_files()?;
Ok(true)
}
fn clean_rewind_files(&self) -> io::Result<u32> {
let data_dir = self.data_dir.clone();
let pattern = format!("{}.", PMMR_LEAF_FILE);
clean_files_by_prefix(data_dir, &pattern, REWIND_FILE_CLEANUP_DURATION_SECONDS)
}
fn pos_to_rm(&self, cutoff_pos: u64, rewind_rm_pos: &Bitmap) -> (Bitmap, Bitmap) {
let mut expanded = Bitmap::new();
let leaf_pos_to_rm =
self.leaf_set
.removed_pre_cutoff(cutoff_pos, rewind_rm_pos, &self.prune_list);
for x in leaf_pos_to_rm.iter() {
expanded.add(x);
let mut current = x as u64;
loop {
let (parent0, sibling0) = family(current - 1);
let sibling_pruned = self.is_pruned_root(sibling0);
if sibling_pruned {
expanded.add(1 + sibling0 as u32);
}
if sibling_pruned || expanded.contains(1 + sibling0 as u32) {
expanded.add(1 + parent0 as u32);
current = 1 + parent0;
} else {
break;
}
}
}
(leaf_pos_to_rm, removed_excl_roots(&expanded))
}
}
fn removed_excl_roots(removed: &Bitmap) -> Bitmap {
removed
.iter()
.filter(|pos| {
let (parent_pos0, _) = family(*pos as u64 - 1);
removed.contains(1 + parent_pos0 as u32)
})
.collect()
}
pub fn clean_files_by_prefix<P: AsRef<std::path::Path>>(
path: P,
prefix_to_delete: &str,
cleanup_duration_seconds: u64,
) -> io::Result<u32> {
let now = time::SystemTime::now();
let cleanup_duration = time::Duration::from_secs(cleanup_duration_seconds);
let number_of_files_deleted: u32 = fs::read_dir(&path)?
.flat_map(
|possible_dir_entry| -> Result<u32, Box<dyn std::error::Error>> {
let dir_entry: std::fs::DirEntry = possible_dir_entry?;
let metadata = dir_entry.metadata()?;
if metadata.is_dir() {
return Ok(0); }
let accessed = metadata.accessed()?;
let duration_since_accessed = now.duration_since(accessed)?;
if duration_since_accessed <= cleanup_duration {
return Ok(0); }
let file_name = dir_entry
.file_name()
.into_string()
.ok()
.ok_or("could not convert filename into utf-8")?;
if file_name.starts_with(prefix_to_delete)
&& file_name.len() > prefix_to_delete.len()
{
if fs::remove_file(dir_entry.path()).is_ok() {
return Ok(1);
}
}
Ok(0)
},
)
.sum();
Ok(number_of_files_deleted)
}