bask 0.2.1

Build Tasks: an async task-queue pipeline engine with pluggable IO, columnar formats, and predefined tasks
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

/*
 * SPDX-FileCopyrightText: 2026 Wavelens GmbH <info@wavelens.io>
 *
 * SPDX-License-Identifier: MIT OR Apache-2.0
 */

//! Convert a CSV file to Parquet through the IO plane: a `SourceWorker` streams the CSV
//! as record batches and a `SinkWorker` writes them out, with the format on each side
//! chosen from its extension by the registry.
//!
//! Run with: cargo run --example io_convert --features formats

use arrow::record_batch::RecordBatch;
use bask::io::{Read, SinkWorker, SourceWorker};
use bask::prelude::*;

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    let dir = std::env::temp_dir();
    let csv = dir.join("bask_io_convert.csv");
    let parquet = dir.join("bask_io_convert.parquet");
    std::fs::write(&csv, "id,name\n1,alice\n2,bob\n3,carol\n")?;

    let sinks = bask::formats::record_sinks();
    let report = Engine::builder()
        .worker(SourceWorker::new(bask::formats::record_sources()))
        .worker_cfg(
            SinkWorker::open(&sinks, parquet.to_str().unwrap())?,
            WorkerCfg::new().label("parquet-out").concurrency(1),
        )
        .seed(Read::<RecordBatch>::new(csv.to_str().unwrap()))
        .run()
        .await?;

    let mut reader = bask::formats::for_path(&parquet)?.open_reader(&parquet, 8192)?;
    let mut rows = 0;
    while let Some(chunk) = reader.next_chunk()? {
        rows += chunk.num_rows();
    }
    std::fs::remove_file(&csv).ok();
    std::fs::remove_file(&parquet).ok();

    println!(
        "csv -> parquet: {rows} rows across {} tasks, {} failures",
        report.stats.processed,
        report.failures.len()
    );
    Ok(())
}