use arrow_array::RecordBatch;
use arrow_schema::SchemaRef;
use bytes::Bytes;
use parquet::arrow::ArrowWriter;
use parquet::basic::{Compression, ZstdLevel};
use parquet::file::properties::WriterProperties;
use crate::adapters::files::_helpers::write::csv_files::{
csv_bytes, csv_header, csv_rows, finish_csv, new_csv, push_csv,
};
use crate::adapters::files::constants::{
CSV_SUFFIX, GZIP_SUFFIX, OPEN_FILE_OVERHEAD_BYTES, PARQUET_SUFFIX, PART_PREFIX, ZSTD_SUFFIX,
};
use crate::adapters::files::errors::FilesDestinationError;
use crate::adapters::files::models::{FileBody, FilesWriter, OpenFile};
use crate::file_store::main::write_file::write_file;
use crate::file_store::models::OpenFolder;
use crate::project::models::{FilesFormat, OutputCompression, OutputEncoding};
pub(crate) fn new_file(schema: &SchemaRef, encoding: &OutputEncoding) -> Result<OpenFile, String> {
let body = match encoding.format {
FilesFormat::Parquet => {
let properties = WriterProperties::builder()
.set_compression(Compression::ZSTD(ZstdLevel::default()))
.set_statistics_truncate_length(None)
.set_column_index_truncate_length(None)
.build();
FileBody::Parquet(Box::new(
ArrowWriter::try_new(Vec::new(), schema.clone(), Some(properties))
.map_err(|error| error.to_string())?,
))
}
FilesFormat::Csv => {
let out = new_csv(encoding.compression);
FileBody::Csv(match encoding.csv.header {
true => push_csv(out, &csv_header(schema, &encoding.csv))?,
false => out,
})
}
};
Ok(OpenFile { body, rows: 0 })
}
pub(crate) fn add_rows(
mut file: OpenFile,
batch: &RecordBatch,
encoding: &OutputEncoding,
) -> Result<OpenFile, String> {
file.body = match file.body {
FileBody::Parquet(mut writer) => {
writer.write(batch).map_err(|error| error.to_string())?;
FileBody::Parquet(writer)
}
FileBody::Csv(out) => FileBody::Csv(push_csv(out, &csv_rows(batch, &encoding.csv)?)?),
};
file.rows += batch.num_rows();
Ok(file)
}
pub(crate) fn open_bytes(file: &OpenFile) -> usize {
let held = match &file.body {
FileBody::Parquet(writer) => {
writer.bytes_written() + writer.in_progress_size().max(writer.memory_size())
}
FileBody::Csv(out) => csv_bytes(out),
};
held + OPEN_FILE_OVERHEAD_BYTES
}
pub(crate) fn next_key(writer: &FilesWriter, value: &str) -> String {
let folder = value_folder(writer, value);
let suffix = file_suffix(&writer.encoding);
match writer.encoding.single_file {
true => format!("{folder}{}{suffix}", writer.single_name),
false => format!(
"{folder}{PART_PREFIX}{}-{}{suffix}",
writer.file_id,
writer.written.len()
),
}
}
pub(crate) fn value_folder(writer: &FilesWriter, value: &str) -> String {
match &writer.partition {
Some(partition) => format!("{}/", partition.name(value)),
None => String::new(),
}
}
pub(crate) fn file_suffix(encoding: &OutputEncoding) -> String {
match (encoding.format, encoding.compression) {
(FilesFormat::Parquet, _) => PARQUET_SUFFIX.to_owned(),
(FilesFormat::Csv, OutputCompression::None) => CSV_SUFFIX.to_owned(),
(FilesFormat::Csv, OutputCompression::Gzip) => format!("{CSV_SUFFIX}{GZIP_SUFFIX}"),
(FilesFormat::Csv, OutputCompression::Zstd) => format!("{CSV_SUFFIX}{ZSTD_SUFFIX}"),
}
}
pub(crate) async fn store_file(
folder: &OpenFolder,
key: &str,
file: OpenFile,
) -> Result<(), FilesDestinationError> {
let failed = |reason: String| FilesDestinationError::Write {
key: key.to_owned(),
reason,
};
let bytes = finish_file(file).map_err(failed)?;
Ok(write_file(folder, key, Bytes::from(bytes)).await?)
}
pub(crate) fn finish_file(file: OpenFile) -> Result<Vec<u8>, String> {
match file.body {
FileBody::Parquet(writer) => writer.into_inner().map_err(|error| error.to_string()),
FileBody::Csv(out) => finish_csv(out),
}
}