parqkit 0.1.0

A fast CLI and Rust library for inspecting, converting, and merging Parquet files.
Documentation
use crate::error::{ParqkitError, ResultExt};
use crate::model::{ColumnInfo, ColumnType, CompressionCodec, CompressionSummary, FileInfo};
use crate::Result;
use arrow::array::RecordBatch;
use arrow::datatypes::{Schema, SchemaRef};
use arrow::record_batch::RecordBatchOptions;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::arrow::ArrowWriter;
use parquet::basic::Compression;
use parquet::file::properties::WriterProperties;
use parquet::file::reader::{FileReader, SerializedFileReader};
use std::fs::{self, File};
use std::path::Path;
use std::sync::Arc;

const ARROW_EXTENSION_TYPE_NAME_KEY: &str = "ARROW:extension:name";
const ARROW_EXTENSION_TYPE_METADATA_KEY: &str = "ARROW:extension:metadata";

pub fn read_head(path: &Path, rows: usize) -> Result<(SchemaRef, Vec<RecordBatch>)> {
    let builder = reader_builder(path)?;
    let schema = schema_without_custom_metadata(builder.schema());

    if rows == 0 {
        return Ok((schema, Vec::new()));
    }

    let reader = builder
        .with_batch_size(rows.min(1024))
        .build()
        .map_err(|error| ParqkitError::from_read(path, error))?;

    let mut batches = Vec::new();
    let mut total_rows = 0usize;

    for batch_result in reader {
        let batch = batch_result.map_err(|error| ParqkitError::corrupted(path, &error))?;
        let rows_needed = rows.saturating_sub(total_rows);
        if rows_needed == 0 {
            break;
        }

        let batch = if batch.num_rows() > rows_needed {
            batch.slice(0, rows_needed)
        } else {
            batch
        };
        let batch = batch_with_schema(path, batch, &schema)?;

        total_rows = total_rows.checked_add(batch.num_rows()).ok_or_else(|| {
            ParqkitError::invalid_metadata(path, "decoded head row count overflow")
        })?;
        batches.push(batch);
    }

    Ok((schema, batches))
}

pub fn read_tail(path: &Path, rows: usize) -> Result<(SchemaRef, Vec<RecordBatch>)> {
    let builder = reader_builder(path)?;
    let schema = schema_without_custom_metadata(builder.schema());

    if rows == 0 {
        return Ok((schema, Vec::new()));
    }

    let metadata = Arc::clone(builder.metadata());
    if metadata.num_row_groups() == 0 {
        return Ok((schema, Vec::new()));
    }

    let (row_groups, rows_to_skip) = tail_row_groups(path, &metadata, rows)?;
    let reader = builder
        .with_row_groups(row_groups)
        .build()
        .map_err(|error| ParqkitError::from_read(path, error))?;

    let mut result_batches = Vec::new();
    let mut skipped = 0usize;

    for batch_result in reader {
        let batch = batch_result.map_err(|error| ParqkitError::corrupted(path, &error))?;

        let batch_end = skipped.checked_add(batch.num_rows()).ok_or_else(|| {
            ParqkitError::invalid_metadata(path, "decoded tail row count overflow")
        })?;
        if batch_end <= rows_to_skip {
            skipped = batch_end;
            continue;
        }

        let offset = rows_to_skip.saturating_sub(skipped);
        let sliced = batch.slice(offset, batch.num_rows() - offset);
        result_batches.push(batch_with_schema(path, sliced, &schema)?);
        skipped = rows_to_skip;
    }

    Ok((schema, result_batches))
}

pub fn row_count(path: &Path) -> Result<i64> {
    let reader = serialized_reader(path)?;
    let rows = reader.metadata().file_metadata().num_rows();
    if rows < 0 {
        return Err(ParqkitError::invalid_metadata(path, "negative row count"));
    }
    Ok(rows)
}

pub fn schema_columns(path: &Path) -> Result<Vec<ColumnInfo>> {
    let reader = serialized_reader(path)?;
    let schema = reader.metadata().file_metadata().schema_descr();

    Ok(schema
        .columns()
        .iter()
        .map(|column| ColumnInfo {
            name: column.path().string(),
            column_type: ColumnType::from_parquet(column),
            nullable: column.max_def_level() > 0,
        })
        .collect())
}

pub fn file_size(path: &Path) -> Result<u64> {
    Ok(fs::metadata(path).with_path_context(path)?.len())
}

pub fn file_info(path: &Path) -> Result<FileInfo> {
    let reader = serialized_reader(path)?;
    let metadata = reader.metadata();
    let file_metadata = metadata.file_metadata();
    let num_rows = file_metadata.num_rows();
    if num_rows < 0 {
        return Err(ParqkitError::invalid_metadata(path, "negative row count"));
    }
    let num_row_groups = metadata.num_row_groups();

    let compression = compression_summary(metadata);

    Ok(FileInfo {
        path: path.to_path_buf(),
        file_size_bytes: file_size(path)?,
        num_rows,
        num_columns: file_metadata.schema_descr().num_columns(),
        num_row_groups,
        compression,
        created_by: file_metadata.created_by().map(ToOwned::to_owned),
        version: file_metadata.version(),
    })
}

pub fn reader_builder(path: &Path) -> Result<ParquetRecordBatchReaderBuilder<File>> {
    let file = File::open(path).with_path_context(path)?;
    ParquetRecordBatchReaderBuilder::try_new(file)
        .map_err(|error| ParqkitError::from_read(path, error))
}

pub fn serialized_reader(path: &Path) -> Result<SerializedFileReader<File>> {
    let file = File::open(path).with_path_context(path)?;
    SerializedFileReader::new(file).map_err(|error| ParqkitError::from_read(path, error))
}

pub fn merge_files(paths: &[&Path], output: &Path) -> Result<()> {
    if paths.is_empty() {
        return Err(ParqkitError::NoInputFiles);
    }

    let first_builder = reader_builder(paths[0])?;
    let schema = schema_without_custom_metadata(first_builder.schema());

    for path in paths.iter().skip(1) {
        let builder = reader_builder(path)?;
        let candidate_schema = schema_without_custom_metadata(builder.schema());
        if candidate_schema.as_ref() != schema.as_ref() {
            return Err(ParqkitError::SchemaMismatch {
                file1: paths[0].display().to_string(),
                file2: path.display().to_string(),
                details: "Column names or types differ".to_string(),
            });
        }
    }

    let mut pending_output = crate::atomic_output::PendingOutput::new(output)?;
    let output_file = pending_output.take_file()?;
    let props = WriterProperties::builder()
        .set_compression(Compression::SNAPPY)
        .build();
    let mut writer = ArrowWriter::try_new(output_file, Arc::clone(&schema), Some(props))
        .map_err(|error| ParqkitError::write_error(output, error))?;

    for path in paths {
        let builder = reader_builder(path)?;
        let reader = builder
            .build()
            .map_err(|error| ParqkitError::from_read(path, error))?;

        for batch_result in reader {
            let batch = batch_result.map_err(|error| ParqkitError::corrupted(path, error))?;
            let batch = batch_with_schema(path, batch, &schema)?;
            writer
                .write(&batch)
                .map_err(|error| ParqkitError::write_error(output, error))?;
        }
    }

    writer
        .close()
        .map_err(|error| ParqkitError::write_error(output, error))?;
    pending_output.commit()
}

pub(crate) fn schema_without_custom_metadata(schema: &SchemaRef) -> SchemaRef {
    let fields = schema
        .fields()
        .iter()
        .map(|field| {
            let metadata = field
                .metadata()
                .iter()
                .filter(|(key, _)| {
                    matches!(
                        key.as_str(),
                        ARROW_EXTENSION_TYPE_NAME_KEY | ARROW_EXTENSION_TYPE_METADATA_KEY
                    )
                })
                .map(|(key, value)| (key.clone(), value.clone()))
                .collect();
            Arc::new(field.as_ref().clone().with_metadata(metadata))
        })
        .collect::<Vec<_>>();
    Arc::new(Schema::new(fields))
}

pub(crate) fn batch_with_schema(
    path: &Path,
    batch: RecordBatch,
    schema: &SchemaRef,
) -> Result<RecordBatch> {
    if schema_without_custom_metadata(&batch.schema()).as_ref() != schema.as_ref() {
        return Err(ParqkitError::invalid_metadata(
            path,
            "decoded batch schema does not match the expected schema",
        ));
    }

    let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows()));
    RecordBatch::try_new_with_options(Arc::clone(schema), batch.columns().to_vec(), &options)
        .map_err(|error| {
            ParqkitError::invalid_metadata(
                path,
                format!("could not normalize decoded batch schema: {error}"),
            )
        })
}

fn tail_row_groups(
    path: &Path,
    metadata: &parquet::file::metadata::ParquetMetaData,
    rows: usize,
) -> Result<(Vec<usize>, usize)> {
    let mut selected_groups = Vec::new();
    let mut selected_rows = 0usize;

    for row_group_index in (0..metadata.num_row_groups()).rev() {
        let row_group = metadata.row_group(row_group_index);
        let row_group_rows = usize::try_from(row_group.num_rows()).map_err(|_| {
            ParqkitError::invalid_metadata(
                path,
                format!(
                    "row group {} in {} cannot be represented on this platform",
                    row_group_index,
                    metadata.file_metadata().schema_descr().root_schema().name()
                ),
            )
        })?;

        selected_groups.push(row_group_index);
        selected_rows = selected_rows.checked_add(row_group_rows).ok_or_else(|| {
            ParqkitError::invalid_metadata(path, "selected tail row count overflow")
        })?;
        if selected_rows >= rows {
            break;
        }
    }

    selected_groups.reverse();
    Ok((selected_groups, selected_rows.saturating_sub(rows)))
}

fn compression_summary(metadata: &parquet::file::metadata::ParquetMetaData) -> CompressionSummary {
    let mut compression = None;

    for row_group_index in 0..metadata.num_row_groups() {
        let row_group = metadata.row_group(row_group_index);
        for column_index in 0..row_group.num_columns() {
            let codec = CompressionCodec::from(row_group.column(column_index).compression());
            match compression {
                None => compression = Some(codec),
                Some(existing) if existing == codec => {}
                Some(_) => return CompressionSummary::Mixed,
            }
        }
    }

    compression.map_or(CompressionSummary::Unknown, CompressionSummary::Single)
}