use arrow_schema::SchemaRef;
use bytes::Bytes;
use parquet::arrow::ArrowWriter;
use parquet::basic::{Compression, ZstdLevel};
use parquet::file::properties::WriterProperties;
use crate::adapters::files::constants::{OPEN_FILE_OVERHEAD_BYTES, PARQUET_SUFFIX, PART_PREFIX};
use crate::adapters::files::errors::FilesDestinationError;
use crate::adapters::files::models::{FilesWriter, OpenFile};
use crate::file_store::main::write_file::write_file;
use crate::file_store::models::OpenFolder;
pub(crate) fn new_file(schema: &SchemaRef) -> Result<OpenFile, String> {
let properties = WriterProperties::builder()
.set_compression(Compression::ZSTD(ZstdLevel::default()))
.set_statistics_truncate_length(None)
.set_column_index_truncate_length(None)
.build();
ArrowWriter::try_new(Vec::new(), schema.clone(), Some(properties))
.map(|writer| OpenFile { writer, rows: 0 })
.map_err(|error| error.to_string())
}
pub(crate) fn open_bytes(file: &OpenFile) -> usize {
file.writer.bytes_written()
+ file
.writer
.in_progress_size()
.max(file.writer.memory_size())
+ OPEN_FILE_OVERHEAD_BYTES
}
pub(crate) fn next_key(writer: &FilesWriter, value: &str) -> String {
let folder = match &writer.partition {
Some(partition) => format!("{}/", partition.name(value)),
None => String::new(),
};
format!(
"{folder}{PART_PREFIX}{}-{}{PARQUET_SUFFIX}",
writer.file_id,
writer.written.len()
)
}
pub(crate) async fn store_file(
folder: &OpenFolder,
key: &str,
file: OpenFile,
) -> Result<(), FilesDestinationError> {
let failed = |error: parquet::errors::ParquetError| FilesDestinationError::Write {
key: key.to_owned(),
reason: error.to_string(),
};
let bytes = file.writer.into_inner().map_err(failed)?;
Ok(write_file(folder, key, Bytes::from(bytes)).await?)
}