use arrow_array::RecordBatch;
use arrow_ipc::CompressionType;
use arrow_ipc::writer::{FileWriter, IpcWriteOptions};
use crate::commit_spool::constants::{FILE_SUFFIX, FOLDER_PREFIX};
use crate::commit_spool::models::{HeldBatch, Spool};
impl Spool {
pub(crate) fn new(parent: &std::path::Path) -> Result<Self, String> {
let folder = tempfile::Builder::new()
.prefix(FOLDER_PREFIX)
.tempdir_in(parent)
.map_err(|error| {
format!(
"cannot make a folder in {} to hold rows until they commit: {error}",
parent.display()
)
})?;
Ok(Self {
folder,
held: Vec::new(),
made: 0,
})
}
pub(crate) fn hold(&mut self, batch: u64, rows: &RecordBatch) -> Result<(), String> {
let path = self
.folder
.path()
.join(format!("{}{FILE_SUFFIX}", self.made));
self.made += 1;
let failed = |error: String| {
format!(
"cannot hold rows in {} until they commit: {error}",
path.display()
)
};
let file = std::fs::File::create(&path).map_err(|error| failed(error.to_string()))?;
let options = IpcWriteOptions::default()
.try_with_compression(Some(CompressionType::ZSTD))
.map_err(|error| failed(error.to_string()))?;
let mut writer = FileWriter::try_new_with_options(
std::io::BufWriter::new(file),
&rows.schema(),
options,
)
.map_err(|error| failed(error.to_string()))?;
writer
.write(rows)
.and_then(|()| writer.finish())
.map_err(|error| failed(error.to_string()))?;
let bytes = std::fs::metadata(&path)
.map(|metadata| usize::try_from(metadata.len()).unwrap_or(usize::MAX))
.map_err(|error| failed(error.to_string()))?;
match self.held.last_mut().filter(|held| held.batch == batch) {
Some(held) => {
held.files.push(path);
held.bytes += bytes;
}
None => self.held.push(HeldBatch {
batch,
files: vec![path],
bytes,
}),
}
Ok(())
}
pub(crate) fn take(&mut self) -> Vec<HeldBatch> {
std::mem::take(&mut self.held)
}
pub(crate) fn take_ended(&mut self, batch: u64) -> Vec<HeldBatch> {
let (ended, open) = std::mem::take(&mut self.held)
.into_iter()
.partition(|held| held.batch < batch);
self.held = open;
ended
}
pub(crate) fn take_open_files(&mut self, batch: u64) -> Vec<std::path::PathBuf> {
match self.held.last_mut().filter(|held| held.batch == batch) {
Some(held) => {
held.bytes = 0;
std::mem::take(&mut held.files)
}
None => Vec::new(),
}
}
pub(crate) fn held_bytes(&self, batch: u64) -> usize {
self.held
.iter()
.filter(|held| held.batch == batch)
.map(|held| held.bytes)
.sum()
}
}