sorting-parquet-writer 0.2.1

A Rust library for writing sorted Parquet files using Apache Arrow.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

use parquet::arrow::ArrowWriter;

use crate::test::{TestArrowType, TickerItem};
fn base_dir() -> std::path::PathBuf {
    let manifest = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
    manifest.join("test_output")
}
#[test]
#[ignore]
fn generate_generic_unsorted_data() -> anyhow::Result<()> {
    let file = std::fs::File::create(base_dir().join("unsorted_generic_data.parquet"))?;
    let props = parquet::file::properties::WriterProperties::builder()
        .set_created_by("sorting-parquet-writer test".to_string())
        .build();
    let schema = TickerItem::schema();
    let mut writer = ArrowWriter::try_new(file, schema, Some(props))?;

    for i in 0..10 {
        eprintln!("Writing batch {}/10", i + 1);
        let items = TickerItem::random_instances(1024 * 1024);
        for chunk in items.chunks(128) {
            let batch = TickerItem::into_record_batch(chunk)?;
            writer.write(&batch)?;
        }
    }
    writer.close()?;
    Ok(())
}