use arrow_array::RecordBatch;
use crate::adapters::files::_helpers::write::data_files::{
add_rows, new_file, next_key, open_bytes, store_file,
};
use crate::adapters::files::_helpers::write::spill::spill_order;
use crate::adapters::files::_helpers::write::split::partition_rows;
use crate::adapters::files::constants::{
FILE_MAX_BYTES, FILE_MAX_ROWS, OPEN_FILES_MAX, OPEN_FILES_MAX_BYTES, SINGLE_FILE_MAX_BYTES,
};
use crate::adapters::files::errors::FilesDestinationError;
use crate::adapters::files::models::FilesWriter;
pub async fn write_rows(
writer: &mut FilesWriter,
batch: RecordBatch,
) -> Result<&mut FilesWriter, FilesDestinationError> {
let parts = partition_rows(writer, &batch).map_err(|reason| FilesDestinationError::Write {
key: writer.folder.location.to_string(),
reason,
})?;
for (value, part) in parts {
let opens = !writer.open.contains_key(&value);
let max_files = match opens {
true => OPEN_FILES_MAX - 1,
false => OPEN_FILES_MAX,
};
spill(writer, max_files).await?;
add_part(writer, value, part).await?;
}
spill(writer, OPEN_FILES_MAX).await
}
async fn spill(
writer: &mut FilesWriter,
max_files: usize,
) -> Result<&mut FilesWriter, FilesDestinationError> {
if writer.encoding.single_file {
let held: usize = writer.open.values().map(open_bytes).sum();
let reason = match (writer.open.len() > max_files, held > SINGLE_FILE_MAX_BYTES) {
(true, _) => format!(
"single_file keeps one file open per partition, and this commit writes more than {OPEN_FILES_MAX} partitions; load fewer partitions at once (--start-cursor/--end-cursor) or drop single_file"
),
(false, true) => format!(
"single_file holds each file in memory until the load commits, and this load passed {} GiB; compress it, partition it, or drop single_file",
SINGLE_FILE_MAX_BYTES / (1024 * 1024 * 1024)
),
(false, false) => return Ok(writer),
};
return Err(FilesDestinationError::Write {
key: writer.folder.location.to_string(),
reason,
});
}
let sizes: Vec<(String, usize)> = writer
.open
.iter()
.map(|(value, file)| (value.clone(), open_bytes(file)))
.collect();
for value in spill_order(&sizes, max_files, OPEN_FILES_MAX_BYTES) {
let Some(file) = writer.open.remove(&value) else {
continue;
};
let key = next_key(writer, &value);
store_file(&writer.folder, &key, file).await?;
writer.written.push(key);
}
Ok(writer)
}
async fn add_part(
writer: &mut FilesWriter,
value: String,
part: RecordBatch,
) -> Result<&mut FilesWriter, FilesDestinationError> {
let failed = |reason: String| FilesDestinationError::Write {
key: writer.folder.location.to_string(),
reason,
};
let file = match writer.open.remove(&value) {
Some(file) => file,
None => new_file(&writer.schema, &writer.encoding).map_err(failed)?,
};
let file = add_rows(file, &part, &writer.encoding).map_err(failed)?;
writer.rows += part.num_rows() as u64;
writer.partitions.insert(value.clone());
let full = file.rows >= FILE_MAX_ROWS || open_bytes(&file) >= FILE_MAX_BYTES;
match full && !writer.encoding.single_file {
true => {
let key = next_key(writer, &value);
store_file(&writer.folder, &key, file).await?;
writer.written.push(key);
}
false => {
writer.open.insert(value, file);
}
}
Ok(writer)
}