use crate::cell::CellValue;
use crate::error::{Result, RustypyxlError};
use crate::worksheet::Worksheet;
use crate::Workbook;
use arrow::array::{
Array, ArrayRef, BooleanArray, Date32Array, Date64Array, Decimal128Array, Decimal256Array,
Float16Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array,
LargeStringArray, StringArray, TimestampMicrosecondArray, TimestampMillisecondArray,
TimestampNanosecondArray, TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array,
UInt8Array,
};
use arrow::datatypes::{DataType, Field, Schema, TimeUnit};
use arrow::record_batch::RecordBatch;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::arrow::ArrowWriter;
use parquet::arrow::ProjectionMask;
use parquet::basic::Compression;
use parquet::file::properties::WriterProperties;
use std::collections::HashMap;
use std::fs::File;
use std::sync::Arc;
#[derive(Debug, Clone)]
pub struct ParquetImportResult {
pub rows_imported: u32,
pub columns_imported: u32,
pub start_row: u32,
pub start_col: u32,
pub end_row: u32,
pub end_col: u32,
pub column_names: Vec<String>,
}
impl ParquetImportResult {
pub fn range_with_headers(&self) -> String {
format!(
"{}{}:{}{}",
crate::utils::column_to_letter(self.start_col),
self.start_row,
crate::utils::column_to_letter(self.end_col),
self.end_row
)
}
pub fn data_range(&self) -> String {
format!(
"{}{}:{}{}",
crate::utils::column_to_letter(self.start_col),
self.start_row + 1,
crate::utils::column_to_letter(self.end_col),
self.end_row
)
}
pub fn header_range(&self) -> String {
format!(
"{}{}:{}{}",
crate::utils::column_to_letter(self.start_col),
self.start_row,
crate::utils::column_to_letter(self.end_col),
self.start_row
)
}
}
#[derive(Debug, Clone)]
pub struct ParquetImportOptions {
pub column_renames: HashMap<String, String>,
pub include_headers: bool,
pub columns: Vec<String>,
pub batch_size: usize,
}
impl Default for ParquetImportOptions {
fn default() -> Self {
Self {
column_renames: HashMap::new(),
include_headers: true,
columns: Vec::new(),
batch_size: 65536,
}
}
}
impl ParquetImportOptions {
pub fn new() -> Self {
Self::default()
}
pub fn rename_column(mut self, from: &str, to: &str) -> Self {
self.column_renames.insert(from.to_string(), to.to_string());
self
}
pub fn with_headers(mut self, include: bool) -> Self {
self.include_headers = include;
self
}
pub fn select_columns(mut self, columns: Vec<String>) -> Self {
self.columns = columns;
self
}
pub fn with_batch_size(mut self, size: usize) -> Self {
self.batch_size = size;
self
}
}
impl Workbook {
pub fn insert_from_parquet(
&mut self,
sheet_name: &str,
path: &str,
start_row: u32,
start_col: u32,
options: Option<ParquetImportOptions>,
) -> Result<ParquetImportResult> {
let options = options.unwrap_or_default();
let opts = if options.batch_size == 0 {
ParquetImportOptions {
batch_size: 65536,
..options
}
} else {
options
};
let file = File::open(path).map_err(|e| {
RustypyxlError::ParseError(format!("Failed to open parquet file: {}", e))
})?;
let builder = ParquetRecordBatchReaderBuilder::try_new(file).map_err(|e| {
RustypyxlError::ParseError(format!("Failed to read parquet metadata: {}", e))
})?;
let schema = builder.schema().clone();
let all_column_names: Vec<String> =
schema.fields().iter().map(|f| f.name().clone()).collect();
let columns_to_import: Vec<usize> = if opts.columns.is_empty() {
(0..all_column_names.len()).collect()
} else {
opts.columns
.iter()
.map(|name| {
all_column_names
.iter()
.position(|n| n == name)
.ok_or_else(|| {
RustypyxlError::ParseError(format!(
"Column '{}' not found in parquet file (available: {})",
name,
all_column_names.join(", ")
))
})
})
.collect::<Result<_>>()?
};
if columns_to_import.is_empty() {
return Err(RustypyxlError::ParseError(
"No matching columns found in parquet file".to_string(),
));
}
let mut projected: Vec<usize> = columns_to_import.clone();
projected.sort_unstable();
projected.dedup();
let batch_indices: Vec<usize> = columns_to_import
.iter()
.map(|idx| projected.iter().position(|p| p == idx).unwrap())
.collect();
let projection = ProjectionMask::roots(builder.parquet_schema(), projected.iter().copied());
let reader = builder
.with_batch_size(opts.batch_size)
.with_projection(projection)
.build()
.map_err(|e| {
RustypyxlError::ParseError(format!("Failed to build parquet reader: {}", e))
})?;
let worksheet = self.get_sheet_by_name_mut(sheet_name)?;
let final_column_names: Vec<String> = columns_to_import
.iter()
.map(|&idx| {
let original = &all_column_names[idx];
opts.column_renames
.get(original)
.cloned()
.unwrap_or_else(|| original.clone())
})
.collect();
let mut current_row = start_row;
if opts.include_headers {
for (col_offset, name) in final_column_names.iter().enumerate() {
let col = start_col + col_offset as u32;
worksheet.set_cell_value(
current_row,
col,
CellValue::String(Arc::from(name.as_str())),
);
}
current_row += 1;
}
let _data_start_row = current_row;
let mut total_rows: u32 = 0;
for batch_result in reader {
let batch = batch_result.map_err(|e| {
RustypyxlError::ParseError(format!("Failed to read parquet batch: {}", e))
})?;
let num_rows = batch.num_rows();
for (col_offset, &batch_idx) in batch_indices.iter().enumerate() {
let col = start_col + col_offset as u32;
let array = batch.column(batch_idx);
write_arrow_array_to_worksheet(worksheet, array, current_row, col, num_rows);
}
current_row += num_rows as u32;
total_rows += num_rows as u32;
}
let end_row_with_header = if opts.include_headers && total_rows > 0 {
start_row + total_rows
} else if total_rows > 0 {
start_row + total_rows - 1
} else {
start_row
};
Ok(ParquetImportResult {
rows_imported: total_rows,
columns_imported: columns_to_import.len() as u32,
start_row,
start_col,
end_row: end_row_with_header,
end_col: start_col + columns_to_import.len() as u32 - 1,
column_names: final_column_names,
})
}
}
const DATE_FORMAT: &str = "yyyy-mm-dd";
const DATETIME_FORMAT: &str = "yyyy-mm-dd hh:mm:ss";
fn date_format() -> &'static crate::cell::InternedString {
static FORMAT: std::sync::OnceLock<crate::cell::InternedString> = std::sync::OnceLock::new();
FORMAT.get_or_init(|| Arc::from(DATE_FORMAT))
}
fn datetime_format() -> &'static crate::cell::InternedString {
static FORMAT: std::sync::OnceLock<crate::cell::InternedString> = std::sync::OnceLock::new();
FORMAT.get_or_init(|| Arc::from(DATETIME_FORMAT))
}
const MAX_EXACT_INT: u64 = 1 << 53;
fn int64_cell_value(value: i64) -> CellValue {
if value.unsigned_abs() <= MAX_EXACT_INT {
CellValue::Number(value as f64)
} else {
CellValue::String(Arc::from(value.to_string()))
}
}
fn uint64_cell_value(value: u64) -> CellValue {
if value <= MAX_EXACT_INT {
CellValue::Number(value as f64)
} else {
CellValue::String(Arc::from(value.to_string()))
}
}
fn set_date_cell(
worksheet: &mut Worksheet,
row: u32,
col: u32,
serial: f64,
format: &crate::cell::InternedString,
) {
let cell = worksheet.get_or_create_cell_mut(row, col);
cell.value = CellValue::Number(serial);
cell.number_format = Some(format.clone());
cell.style_index = None;
}
fn write_arrow_array_to_worksheet(
worksheet: &mut Worksheet,
array: &ArrayRef,
start_row: u32,
col: u32,
num_rows: usize,
) {
match array.data_type() {
DataType::Null => {
}
DataType::Boolean => {
let arr = array.as_any().downcast_ref::<BooleanArray>().unwrap();
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
worksheet.set_cell_value(row, col, CellValue::Boolean(arr.value(i)));
}
}
}
DataType::Int8 => write_int_array(worksheet, array, start_row, col, num_rows),
DataType::Int16 => write_int_array(worksheet, array, start_row, col, num_rows),
DataType::Int32 => write_int_array(worksheet, array, start_row, col, num_rows),
DataType::Int64 => write_int_array(worksheet, array, start_row, col, num_rows),
DataType::UInt8 => write_uint_array(worksheet, array, start_row, col, num_rows),
DataType::UInt16 => write_uint_array(worksheet, array, start_row, col, num_rows),
DataType::UInt32 => write_uint_array(worksheet, array, start_row, col, num_rows),
DataType::UInt64 => write_uint_array(worksheet, array, start_row, col, num_rows),
DataType::Float16 => {
let arr = array.as_any().downcast_ref::<Float16Array>().unwrap();
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
worksheet.set_cell_value(row, col, CellValue::Number(arr.value(i).to_f64()));
}
}
}
DataType::Float32 => {
let arr = array.as_any().downcast_ref::<Float32Array>().unwrap();
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
worksheet.set_cell_value(row, col, CellValue::Number(arr.value(i) as f64));
}
}
}
DataType::Float64 => {
let arr = array.as_any().downcast_ref::<Float64Array>().unwrap();
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
worksheet.set_cell_value(row, col, CellValue::Number(arr.value(i)));
}
}
}
DataType::Utf8 => {
let arr = array.as_any().downcast_ref::<StringArray>().unwrap();
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
worksheet.set_cell_value(row, col, CellValue::String(Arc::from(arr.value(i))));
}
}
}
DataType::LargeUtf8 => {
let arr = array.as_any().downcast_ref::<LargeStringArray>().unwrap();
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
worksheet.set_cell_value(row, col, CellValue::String(Arc::from(arr.value(i))));
}
}
}
DataType::Date32 => {
let arr = array.as_any().downcast_ref::<Date32Array>().unwrap();
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
let days = arr.value(i);
let excel_serial = days + 25569;
set_date_cell(worksheet, row, col, excel_serial as f64, date_format());
}
}
}
DataType::Date64 => {
let arr = array.as_any().downcast_ref::<Date64Array>().unwrap();
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
let ms = arr.value(i);
let days = ms as f64 / (24.0 * 60.0 * 60.0 * 1000.0);
let excel_serial = days + 25569.0;
set_date_cell(worksheet, row, col, excel_serial, date_format());
}
}
}
DataType::Timestamp(unit, _tz) => match unit {
TimeUnit::Second => {
let arr = array
.as_any()
.downcast_ref::<TimestampSecondArray>()
.unwrap();
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
let secs = arr.value(i) as f64;
let days = secs / (24.0 * 60.0 * 60.0);
let excel_serial = days + 25569.0;
set_date_cell(worksheet, row, col, excel_serial, datetime_format());
}
}
}
TimeUnit::Millisecond => {
let arr = array
.as_any()
.downcast_ref::<TimestampMillisecondArray>()
.unwrap();
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
let ms = arr.value(i) as f64;
let days = ms / (24.0 * 60.0 * 60.0 * 1000.0);
let excel_serial = days + 25569.0;
set_date_cell(worksheet, row, col, excel_serial, datetime_format());
}
}
}
TimeUnit::Microsecond => {
let arr = array
.as_any()
.downcast_ref::<TimestampMicrosecondArray>()
.unwrap();
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
let us = arr.value(i) as f64;
let days = us / (24.0 * 60.0 * 60.0 * 1_000_000.0);
let excel_serial = days + 25569.0;
set_date_cell(worksheet, row, col, excel_serial, datetime_format());
}
}
}
TimeUnit::Nanosecond => {
let arr = array
.as_any()
.downcast_ref::<TimestampNanosecondArray>()
.unwrap();
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
let ns = arr.value(i) as f64;
let days = ns / (24.0 * 60.0 * 60.0 * 1_000_000_000.0);
let excel_serial = days + 25569.0;
set_date_cell(worksheet, row, col, excel_serial, datetime_format());
}
}
}
},
DataType::Decimal128(_, scale) => {
let arr = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
let scale_factor = 10f64.powi(*scale as i32);
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
let val = arr.value(i) as f64 / scale_factor;
worksheet.set_cell_value(row, col, CellValue::Number(val));
}
}
}
DataType::Decimal256(_, scale) => {
let arr = array.as_any().downcast_ref::<Decimal256Array>().unwrap();
let scale_factor = 10f64.powi(*scale as i32);
for i in 0..num_rows {
let row = start_row + i as u32;
if arr.is_valid(i) {
let val = arr
.value(i)
.to_string()
.parse::<f64>()
.map(|v| v / scale_factor)
.unwrap_or(f64::NAN);
worksheet.set_cell_value(row, col, CellValue::Number(val));
}
}
}
_ => {
let formatter = arrow::util::display::ArrayFormatter::try_new(
array.as_ref(),
&arrow::util::display::FormatOptions::default(),
);
if let Ok(fmt) = formatter {
for i in 0..num_rows {
let row = start_row + i as u32;
if array.is_valid(i) {
let s = fmt.value(i).to_string();
worksheet.set_cell_value(row, col, CellValue::String(Arc::from(s)));
}
}
}
}
}
}
fn write_int_array(
worksheet: &mut Worksheet,
array: &ArrayRef,
start_row: u32,
col: u32,
num_rows: usize,
) {
if let Some(arr) = array.as_any().downcast_ref::<Int8Array>() {
for i in 0..num_rows {
if arr.is_valid(i) {
worksheet.set_cell_value(
start_row + i as u32,
col,
CellValue::Number(arr.value(i) as f64),
);
}
}
} else if let Some(arr) = array.as_any().downcast_ref::<Int16Array>() {
for i in 0..num_rows {
if arr.is_valid(i) {
worksheet.set_cell_value(
start_row + i as u32,
col,
CellValue::Number(arr.value(i) as f64),
);
}
}
} else if let Some(arr) = array.as_any().downcast_ref::<Int32Array>() {
for i in 0..num_rows {
if arr.is_valid(i) {
worksheet.set_cell_value(
start_row + i as u32,
col,
CellValue::Number(arr.value(i) as f64),
);
}
}
} else if let Some(arr) = array.as_any().downcast_ref::<Int64Array>() {
for i in 0..num_rows {
if arr.is_valid(i) {
worksheet.set_cell_value(start_row + i as u32, col, int64_cell_value(arr.value(i)));
}
}
}
}
fn write_uint_array(
worksheet: &mut Worksheet,
array: &ArrayRef,
start_row: u32,
col: u32,
num_rows: usize,
) {
if let Some(arr) = array.as_any().downcast_ref::<UInt8Array>() {
for i in 0..num_rows {
if arr.is_valid(i) {
worksheet.set_cell_value(
start_row + i as u32,
col,
CellValue::Number(arr.value(i) as f64),
);
}
}
} else if let Some(arr) = array.as_any().downcast_ref::<UInt16Array>() {
for i in 0..num_rows {
if arr.is_valid(i) {
worksheet.set_cell_value(
start_row + i as u32,
col,
CellValue::Number(arr.value(i) as f64),
);
}
}
} else if let Some(arr) = array.as_any().downcast_ref::<UInt32Array>() {
for i in 0..num_rows {
if arr.is_valid(i) {
worksheet.set_cell_value(
start_row + i as u32,
col,
CellValue::Number(arr.value(i) as f64),
);
}
}
} else if let Some(arr) = array.as_any().downcast_ref::<UInt64Array>() {
for i in 0..num_rows {
if arr.is_valid(i) {
worksheet.set_cell_value(
start_row + i as u32,
col,
uint64_cell_value(arr.value(i)),
);
}
}
}
}
#[derive(Debug, Clone)]
pub struct ParquetExportResult {
pub rows_exported: u32,
pub columns_exported: u32,
pub column_names: Vec<String>,
pub file_size: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ColumnType {
#[default]
Auto,
String,
Float64,
Int64,
Boolean,
Date,
DateTime,
}
#[derive(Debug, Clone)]
pub struct ParquetExportOptions {
pub column_renames: HashMap<String, String>,
pub has_headers: bool,
pub compression: ParquetCompression,
pub column_types: HashMap<String, ColumnType>,
pub row_group_size: usize,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum ParquetCompression {
None,
#[default]
Snappy,
Gzip,
Zstd,
Lz4,
}
impl From<ParquetCompression> for Compression {
fn from(c: ParquetCompression) -> Self {
match c {
ParquetCompression::None => Compression::UNCOMPRESSED,
ParquetCompression::Snappy => Compression::SNAPPY,
ParquetCompression::Gzip => Compression::GZIP(Default::default()),
ParquetCompression::Zstd => Compression::ZSTD(Default::default()),
ParquetCompression::Lz4 => Compression::LZ4,
}
}
}
impl Default for ParquetExportOptions {
fn default() -> Self {
Self {
column_renames: HashMap::new(),
has_headers: true,
compression: ParquetCompression::default(),
column_types: HashMap::new(),
row_group_size: 65536,
}
}
}
impl ParquetExportOptions {
pub fn new() -> Self {
Self::default()
}
pub fn with_headers(mut self, has_headers: bool) -> Self {
self.has_headers = has_headers;
self
}
pub fn rename_column(mut self, from: &str, to: &str) -> Self {
self.column_renames.insert(from.to_string(), to.to_string());
self
}
pub fn with_compression(mut self, compression: ParquetCompression) -> Self {
self.compression = compression;
self
}
pub fn with_column_type(mut self, column: &str, col_type: ColumnType) -> Self {
self.column_types.insert(column.to_string(), col_type);
self
}
pub fn with_row_group_size(mut self, size: usize) -> Self {
self.row_group_size = size;
self
}
}
impl Workbook {
pub fn export_to_parquet(
&self,
sheet_name: &str,
path: &str,
options: Option<ParquetExportOptions>,
) -> Result<ParquetExportResult> {
let worksheet = self.get_sheet_by_name(sheet_name)?;
let (min_row, min_col, max_row, max_col) = worksheet.dimensions();
if max_row < min_row || max_col < min_col {
return Err(RustypyxlError::custom("Worksheet is empty"));
}
self.export_cells(
sheet_name,
path,
min_row,
min_col,
max_row,
max_col,
options.unwrap_or_default(),
)
}
#[allow(clippy::too_many_arguments)]
pub fn export_range_to_parquet(
&self,
sheet_name: &str,
path: &str,
min_row: u32,
min_col: u32,
max_row: u32,
max_col: u32,
options: Option<ParquetExportOptions>,
) -> Result<ParquetExportResult> {
if max_row < min_row || max_col < min_col {
return Err(RustypyxlError::custom("Invalid range"));
}
self.export_cells(
sheet_name,
path,
min_row,
min_col,
max_row,
max_col,
options.unwrap_or_default(),
)
}
#[allow(clippy::too_many_arguments)]
fn export_cells(
&self,
sheet_name: &str,
path: &str,
min_row: u32,
min_col: u32,
max_row: u32,
max_col: u32,
options: ParquetExportOptions,
) -> Result<ParquetExportResult> {
let worksheet = self.get_sheet_by_name(sheet_name)?;
let num_cols = (max_col - min_col + 1) as usize;
let data_start_row = if options.has_headers {
min_row + 1
} else {
min_row
};
let num_data_rows = if max_row >= data_start_row {
(max_row - data_start_row + 1) as usize
} else {
0
};
let column_names: Vec<String> = if options.has_headers {
(min_col..=max_col)
.map(|col| {
let original = worksheet
.get_cell_value(min_row, col)
.map(|v| v.to_string())
.unwrap_or_else(|| format!("Column{}", col - min_col + 1));
options
.column_renames
.get(&original)
.cloned()
.unwrap_or(original)
})
.collect()
} else {
(min_col..=max_col)
.map(|col| format!("Column{}", col - min_col + 1))
.collect()
};
let resolved_types: Vec<ColumnType> = column_names
.iter()
.enumerate()
.map(|(col_idx, col_name)| {
let hint = options
.column_types
.get(col_name)
.copied()
.unwrap_or(ColumnType::Auto);
if hint != ColumnType::Auto || num_data_rows == 0 {
return hint;
}
let col = min_col + col_idx as u32;
let col_data: Vec<Option<&CellValue>> = (data_start_row..=max_row)
.map(|row| worksheet.get_cell_value(row, col))
.collect();
infer_column_type(&col_data)
})
.collect();
let fields: Vec<Field> = column_names
.iter()
.zip(&resolved_types)
.map(|(name, col_type)| {
let (field, _) = build_arrow_column(name, &[], *col_type);
field
})
.collect();
let schema = Arc::new(Schema::new(fields));
let file = File::create(path)
.map_err(|e| RustypyxlError::custom(format!("Failed to create file: {}", e)))?;
let props = WriterProperties::builder()
.set_compression(options.compression.into())
.set_max_row_group_size(options.row_group_size)
.build();
let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).map_err(|e| {
RustypyxlError::custom(format!("Failed to create parquet writer: {}", e))
})?;
let chunk_rows = options.row_group_size.max(1) as u32;
let mut chunk_start = data_start_row;
while chunk_start <= max_row && num_data_rows > 0 {
let chunk_end = chunk_start.saturating_add(chunk_rows - 1).min(max_row);
let mut arrays: Vec<ArrayRef> = Vec::with_capacity(num_cols);
for (col_idx, col_name) in column_names.iter().enumerate() {
let col = min_col + col_idx as u32;
let col_data: Vec<Option<&CellValue>> = (chunk_start..=chunk_end)
.map(|row| worksheet.get_cell_value(row, col))
.collect();
let (_, array) = build_arrow_column(col_name, &col_data, resolved_types[col_idx]);
arrays.push(array);
}
let batch = RecordBatch::try_new(schema.clone(), arrays).map_err(|e| {
RustypyxlError::custom(format!("Failed to create record batch: {}", e))
})?;
writer
.write(&batch)
.map_err(|e| RustypyxlError::custom(format!("Failed to write batch: {}", e)))?;
chunk_start = chunk_end + 1;
}
writer
.close()
.map_err(|e| RustypyxlError::custom(format!("Failed to close writer: {}", e)))?;
let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
Ok(ParquetExportResult {
rows_exported: num_data_rows as u32,
columns_exported: num_cols as u32,
column_names,
file_size,
})
}
}
fn infer_column_type(values: &[Option<&CellValue>]) -> ColumnType {
let mut has_string = false;
let mut has_number = false;
let mut has_boolean = false;
let mut all_integers = true;
for value in values.iter().flatten() {
match value {
CellValue::String(_) | CellValue::Formula(_) | CellValue::Date(_) => {
has_string = true;
}
CellValue::Number(n) => {
has_number = true;
if n.fract() != 0.0 {
all_integers = false;
}
}
CellValue::Boolean(_) => {
has_boolean = true;
}
CellValue::Empty => {}
}
}
if has_string {
ColumnType::String
} else if has_number {
if all_integers {
ColumnType::Int64
} else {
ColumnType::Float64
}
} else if has_boolean {
ColumnType::Boolean
} else {
ColumnType::String }
}
fn build_arrow_column(
name: &str,
values: &[Option<&CellValue>],
type_hint: ColumnType,
) -> (Field, ArrayRef) {
let col_type = if type_hint == ColumnType::Auto {
infer_column_type(values)
} else {
type_hint
};
match col_type {
ColumnType::String | ColumnType::Auto => {
let arr: StringArray = values.iter().map(|v| v.map(|cv| cv.to_string())).collect();
(
Field::new(name, DataType::Utf8, true),
Arc::new(arr) as ArrayRef,
)
}
ColumnType::Float64 => {
let arr: Float64Array = values
.iter()
.map(|v| v.and_then(cell_value_to_f64))
.collect();
(
Field::new(name, DataType::Float64, true),
Arc::new(arr) as ArrayRef,
)
}
ColumnType::Int64 => {
let arr: Int64Array = values
.iter()
.map(|v| v.and_then(cell_value_to_i64))
.collect();
(
Field::new(name, DataType::Int64, true),
Arc::new(arr) as ArrayRef,
)
}
ColumnType::Boolean => {
let arr: BooleanArray = values
.iter()
.map(|v| v.and_then(cell_value_to_bool))
.collect();
(
Field::new(name, DataType::Boolean, true),
Arc::new(arr) as ArrayRef,
)
}
ColumnType::Date => {
let arr: Date32Array = values
.iter()
.map(|v| v.and_then(cell_value_to_date32))
.collect();
(
Field::new(name, DataType::Date32, true),
Arc::new(arr) as ArrayRef,
)
}
ColumnType::DateTime => {
let arr: TimestampMillisecondArray = values
.iter()
.map(|v| v.and_then(cell_value_to_timestamp_ms))
.collect();
(
Field::new(name, DataType::Timestamp(TimeUnit::Millisecond, None), true),
Arc::new(arr) as ArrayRef,
)
}
}
}
fn cell_value_to_f64(value: &CellValue) -> Option<f64> {
match value {
CellValue::Number(n) => Some(*n),
CellValue::Boolean(b) => Some(if *b { 1.0 } else { 0.0 }),
CellValue::String(s) => s.parse().ok(),
CellValue::Formula(s) => s.parse().ok(),
_ => None,
}
}
fn cell_value_to_i64(value: &CellValue) -> Option<i64> {
match value {
CellValue::Number(n) => Some(*n as i64),
CellValue::Boolean(b) => Some(if *b { 1 } else { 0 }),
CellValue::String(s) => s.parse().ok(),
CellValue::Formula(s) => s.parse().ok(),
_ => None,
}
}
fn cell_value_to_bool(value: &CellValue) -> Option<bool> {
match value {
CellValue::Boolean(b) => Some(*b),
CellValue::Number(n) => Some(*n != 0.0),
CellValue::String(s) => {
let lower = s.to_lowercase();
if lower == "true" || lower == "yes" || lower == "1" {
Some(true)
} else if lower == "false" || lower == "no" || lower == "0" {
Some(false)
} else {
None
}
}
_ => None,
}
}
fn cell_value_to_date32(value: &CellValue) -> Option<i32> {
match value {
CellValue::Number(n) => {
Some((*n as i32) - 25569)
}
_ => None,
}
}
fn cell_value_to_timestamp_ms(value: &CellValue) -> Option<i64> {
match value {
CellValue::Number(n) => {
let days_since_unix = *n - 25569.0;
let ms = days_since_unix * 24.0 * 60.0 * 60.0 * 1000.0;
Some(ms as i64)
}
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::NamedTempFile;
fn import_column(field: Field, array: ArrayRef) -> Workbook {
let schema = Arc::new(Schema::new(vec![field]));
let batch = RecordBatch::try_new(schema.clone(), vec![array]).unwrap();
let file = NamedTempFile::new().unwrap();
let mut writer = ArrowWriter::try_new(file.reopen().unwrap(), schema, None).unwrap();
writer.write(&batch).unwrap();
writer.close().unwrap();
let mut wb = Workbook::new();
wb.create_sheet(Some("Data".to_string())).unwrap();
wb.insert_from_parquet("Data", file.path().to_str().unwrap(), 1, 1, None)
.unwrap();
wb
}
#[test]
fn test_int64_beyond_f64_precision_is_kept_exact() {
let values = Int64Array::from(vec![
42i64,
9_007_199_254_740_992, 9_007_199_254_740_993, -9_007_199_254_740_993, ]);
let wb = import_column(
Field::new("id", DataType::Int64, false),
Arc::new(values) as ArrayRef,
);
let ws = wb.get_sheet_by_name("Data").unwrap();
assert_eq!(ws.get_cell_value(2, 1), Some(&CellValue::Number(42.0)));
assert_eq!(
ws.get_cell_value(3, 1),
Some(&CellValue::Number(9_007_199_254_740_992.0))
);
assert_eq!(
ws.get_cell_value(4, 1),
Some(&CellValue::String(Arc::from("9007199254740993"))),
"a value past 2^53 must keep its exact digits"
);
assert_eq!(
ws.get_cell_value(5, 1),
Some(&CellValue::String(Arc::from("-9007199254740993")))
);
}
#[test]
fn test_uint64_beyond_f64_precision_is_kept_exact() {
let values = UInt64Array::from(vec![
7u64,
9_007_199_254_740_993,
u64::MAX, ]);
let wb = import_column(
Field::new("id", DataType::UInt64, false),
Arc::new(values) as ArrayRef,
);
let ws = wb.get_sheet_by_name("Data").unwrap();
assert_eq!(ws.get_cell_value(2, 1), Some(&CellValue::Number(7.0)));
assert_eq!(
ws.get_cell_value(3, 1),
Some(&CellValue::String(Arc::from("9007199254740993")))
);
assert_eq!(
ws.get_cell_value(4, 1),
Some(&CellValue::String(Arc::from("18446744073709551615")))
);
}
#[test]
fn test_int32_stays_numeric() {
let values = Int32Array::from(vec![i32::MIN, 0, i32::MAX]);
let wb = import_column(
Field::new("n", DataType::Int32, false),
Arc::new(values) as ArrayRef,
);
let ws = wb.get_sheet_by_name("Data").unwrap();
assert_eq!(
ws.get_cell_value(2, 1),
Some(&CellValue::Number(i32::MIN as f64))
);
assert_eq!(
ws.get_cell_value(4, 1),
Some(&CellValue::Number(i32::MAX as f64))
);
}
#[test]
fn test_import_options_builder() {
let opts = ParquetImportOptions::new()
.rename_column("old_name", "new_name")
.with_headers(true)
.select_columns(vec!["col1".to_string(), "col2".to_string()])
.with_batch_size(1000);
assert_eq!(
opts.column_renames.get("old_name"),
Some(&"new_name".to_string())
);
assert!(opts.include_headers);
assert_eq!(opts.columns, vec!["col1", "col2"]);
assert_eq!(opts.batch_size, 1000);
}
#[test]
fn test_import_result_ranges() {
let result = ParquetImportResult {
rows_imported: 100,
columns_imported: 5,
start_row: 1,
start_col: 1,
end_row: 101,
end_col: 5,
column_names: vec!["A".into(), "B".into(), "C".into(), "D".into(), "E".into()],
};
assert_eq!(result.range_with_headers(), "A1:E101");
assert_eq!(result.data_range(), "A2:E101");
assert_eq!(result.header_range(), "A1:E1");
}
#[test]
fn test_export_options_builder() {
let opts = ParquetExportOptions::new()
.rename_column("old_name", "new_name")
.with_headers(true)
.with_compression(ParquetCompression::Zstd)
.with_column_type("numbers", ColumnType::Float64)
.with_row_group_size(10000);
assert_eq!(
opts.column_renames.get("old_name"),
Some(&"new_name".to_string())
);
assert!(opts.has_headers);
assert_eq!(opts.compression, ParquetCompression::Zstd);
assert_eq!(opts.column_types.get("numbers"), Some(&ColumnType::Float64));
assert_eq!(opts.row_group_size, 10000);
}
#[test]
fn test_infer_column_type_numbers() {
let v1 = CellValue::Number(1.0);
let v2 = CellValue::Number(2.0);
let v3 = CellValue::Number(3.0);
let values: Vec<Option<&CellValue>> = vec![Some(&v1), Some(&v2), Some(&v3)];
assert_eq!(infer_column_type(&values), ColumnType::Int64);
let v4 = CellValue::Number(1.5);
let values2: Vec<Option<&CellValue>> = vec![Some(&v1), Some(&v4)];
assert_eq!(infer_column_type(&values2), ColumnType::Float64);
}
#[test]
fn test_infer_column_type_strings() {
let v1 = CellValue::String(Arc::from("hello"));
let v2 = CellValue::Number(42.0);
let values: Vec<Option<&CellValue>> = vec![Some(&v1), Some(&v2)];
assert_eq!(infer_column_type(&values), ColumnType::String);
}
#[test]
fn test_infer_column_type_booleans() {
let v1 = CellValue::Boolean(true);
let v2 = CellValue::Boolean(false);
let values: Vec<Option<&CellValue>> = vec![Some(&v1), Some(&v2)];
assert_eq!(infer_column_type(&values), ColumnType::Boolean);
}
#[test]
fn test_export_chunks_rows_into_row_groups() {
let mut wb = Workbook::new();
wb.create_sheet(Some("Big".to_string())).unwrap();
{
let ws = wb.get_sheet_by_name_mut("Big").unwrap();
ws.set_cell_value(1, 1, CellValue::from("n"));
for r in 0..100u32 {
ws.set_cell_value(r + 2, 1, CellValue::Number(r as f64));
}
}
let dir = std::env::temp_dir().join("rustypyxl_parquet_tests");
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("chunked.parquet");
let path_str = path.to_str().unwrap();
let opts = ParquetExportOptions {
row_group_size: 10,
..Default::default()
};
let result = wb.export_to_parquet("Big", path_str, Some(opts)).unwrap();
assert_eq!(result.rows_exported, 100);
let file = File::open(&path).unwrap();
let builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
assert_eq!(builder.metadata().num_row_groups(), 10);
let reader = builder.build().unwrap();
let mut values = Vec::new();
for batch in reader {
let batch = batch.unwrap();
let arr = batch
.column(0)
.as_any()
.downcast_ref::<Int64Array>()
.unwrap();
for i in 0..arr.len() {
values.push(arr.value(i) as f64);
}
}
assert_eq!(values.len(), 100);
assert_eq!(values[0], 0.0);
assert_eq!(values[99], 99.0);
std::fs::remove_file(&path).ok();
}
#[test]
fn test_export_roundtrip() {
let mut wb = Workbook::new();
wb.create_sheet(Some("TestSheet".to_string())).unwrap();
wb.set_cell_value_in_sheet("TestSheet", 1, 1, CellValue::String(Arc::from("Name")))
.unwrap();
wb.set_cell_value_in_sheet("TestSheet", 1, 2, CellValue::String(Arc::from("Age")))
.unwrap();
wb.set_cell_value_in_sheet("TestSheet", 1, 3, CellValue::String(Arc::from("Score")))
.unwrap();
wb.set_cell_value_in_sheet("TestSheet", 2, 1, CellValue::String(Arc::from("Alice")))
.unwrap();
wb.set_cell_value_in_sheet("TestSheet", 2, 2, CellValue::Number(30.0))
.unwrap();
wb.set_cell_value_in_sheet("TestSheet", 2, 3, CellValue::Number(95.5))
.unwrap();
wb.set_cell_value_in_sheet("TestSheet", 3, 1, CellValue::String(Arc::from("Bob")))
.unwrap();
wb.set_cell_value_in_sheet("TestSheet", 3, 2, CellValue::Number(25.0))
.unwrap();
wb.set_cell_value_in_sheet("TestSheet", 3, 3, CellValue::Number(87.3))
.unwrap();
let temp = NamedTempFile::new().unwrap();
let path = temp.path().to_str().unwrap();
let result = wb.export_to_parquet("TestSheet", path, None).unwrap();
assert_eq!(result.rows_exported, 2);
assert_eq!(result.columns_exported, 3);
assert_eq!(result.column_names, vec!["Name", "Age", "Score"]);
assert!(result.file_size > 0);
let mut wb2 = Workbook::new();
wb2.create_sheet(Some("Imported".to_string())).unwrap();
let import_result = wb2
.insert_from_parquet("Imported", path, 1, 1, None)
.unwrap();
assert_eq!(import_result.rows_imported, 2);
assert_eq!(import_result.columns_imported, 3);
let ws = wb2.get_sheet_by_name("Imported").unwrap();
assert_eq!(
ws.get_cell_value(1, 1),
Some(&CellValue::String(Arc::from("Name")))
);
assert_eq!(
ws.get_cell_value(2, 1),
Some(&CellValue::String(Arc::from("Alice")))
);
assert_eq!(
ws.get_cell_value(3, 1),
Some(&CellValue::String(Arc::from("Bob")))
);
}
#[test]
fn test_parquet_roundtrip_parquet_to_sheet_to_parquet() {
use arrow::datatypes::Schema;
use arrow::record_batch::RecordBatch;
use parquet::arrow::ArrowWriter;
let temp_parquet1 = NamedTempFile::new().unwrap();
let temp_parquet2 = NamedTempFile::new().unwrap();
let path1 = temp_parquet1.path().to_str().unwrap();
let path2 = temp_parquet2.path().to_str().unwrap();
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("name", DataType::Utf8, true),
Field::new("value", DataType::Float64, true),
Field::new("active", DataType::Boolean, true),
]));
let id_array = Int64Array::from(vec![1, 2, 3, 4, 5]);
let name_array = StringArray::from(vec![
Some("Alice"),
Some("Bob"),
Some("Charlie"),
None,
Some("Eve"),
]);
let value_array = Float64Array::from(vec![
Some(100.5),
Some(200.0),
None,
Some(400.25),
Some(500.75),
]);
let active_array =
BooleanArray::from(vec![Some(true), Some(false), Some(true), None, Some(false)]);
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(id_array),
Arc::new(name_array),
Arc::new(value_array),
Arc::new(active_array),
],
)
.unwrap();
let file = File::create(path1).unwrap();
let mut writer = ArrowWriter::try_new(file, schema, None).unwrap();
writer.write(&batch).unwrap();
writer.close().unwrap();
let mut wb = Workbook::new();
wb.create_sheet(Some("Data".to_string())).unwrap();
let import_result = wb.insert_from_parquet("Data", path1, 1, 1, None).unwrap();
assert_eq!(import_result.rows_imported, 5);
assert_eq!(import_result.columns_imported, 4);
let export_result = wb.export_to_parquet("Data", path2, None).unwrap();
assert_eq!(export_result.rows_exported, 5);
assert_eq!(export_result.columns_exported, 4);
let mut wb2 = Workbook::new();
wb2.create_sheet(Some("Imported".to_string())).unwrap();
let import_result2 = wb2
.insert_from_parquet("Imported", path2, 1, 1, None)
.unwrap();
assert_eq!(import_result2.rows_imported, 5);
assert_eq!(import_result2.columns_imported, 4);
let ws1 = wb.get_sheet_by_name("Data").unwrap();
let ws2 = wb2.get_sheet_by_name("Imported").unwrap();
assert_eq!(
ws1.get_cell_value(1, 1).map(|v| v.to_string()),
ws2.get_cell_value(1, 1).map(|v| v.to_string())
);
assert_eq!(
ws1.get_cell_value(1, 2).map(|v| v.to_string()),
ws2.get_cell_value(1, 2).map(|v| v.to_string())
);
assert_eq!(
ws1.get_cell_value(1, 3).map(|v| v.to_string()),
ws2.get_cell_value(1, 3).map(|v| v.to_string())
);
assert_eq!(
ws1.get_cell_value(1, 4).map(|v| v.to_string()),
ws2.get_cell_value(1, 4).map(|v| v.to_string())
);
for row in 2..=6 {
for col in 1..=4 {
let v1 = ws1.get_cell_value(row, col).map(|v| v.to_string());
let v2 = ws2.get_cell_value(row, col).map(|v| v.to_string());
assert_eq!(v1, v2, "Mismatch at row {} col {}", row, col);
}
}
}
#[test]
fn test_parquet_compression_options() {
let temp = NamedTempFile::new().unwrap();
let path = temp.path().to_str().unwrap();
let mut wb = Workbook::new();
wb.create_sheet(Some("Data".to_string())).unwrap();
wb.set_cell_value_in_sheet("Data", 1, 1, CellValue::String(Arc::from("Col1")))
.unwrap();
wb.set_cell_value_in_sheet("Data", 2, 1, CellValue::Number(42.0))
.unwrap();
let opts_zstd = ParquetExportOptions::new().with_compression(ParquetCompression::Zstd);
let result = wb.export_to_parquet("Data", path, Some(opts_zstd)).unwrap();
assert!(result.file_size > 0);
let opts_none = ParquetExportOptions::new().with_compression(ParquetCompression::None);
let result = wb.export_to_parquet("Data", path, Some(opts_none)).unwrap();
assert!(result.file_size > 0);
}
#[test]
fn test_parquet_column_type_hints() {
let temp = NamedTempFile::new().unwrap();
let path = temp.path().to_str().unwrap();
let mut wb = Workbook::new();
wb.create_sheet(Some("Data".to_string())).unwrap();
wb.set_cell_value_in_sheet("Data", 1, 1, CellValue::String(Arc::from("Value")))
.unwrap();
wb.set_cell_value_in_sheet("Data", 2, 1, CellValue::Number(1.0))
.unwrap();
wb.set_cell_value_in_sheet("Data", 3, 1, CellValue::Number(2.0))
.unwrap();
let opts = ParquetExportOptions::new().with_column_type("Value", ColumnType::Float64);
let result = wb.export_to_parquet("Data", path, Some(opts)).unwrap();
assert_eq!(result.rows_exported, 2);
assert!(result.file_size > 0);
}
}