quiver 0.2.0

A schema specification and validator for Arrow record batches
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

//! A simple example of `#[derive(Quiver)]`, mixing strongly-typed quiver columns
//! with dynamically-typed raw arrow columns.
//!
//! Run with: `cargo run --example example`

#![expect(clippy::print_stdout)] // This is an example, after all

use std::collections::BTreeMap;
use std::sync::Arc;

use quiver::arrow::array::{ArrayRef, StringArray};
use quiver::{Column, Quiver, Utf8};

/// A set of measurements.
#[derive(Quiver)]
struct Measurements {
    /// Metadata of the record batch.
    #[quiver(metadata)]
    metadata: BTreeMap<String, String>,

    /// Name of the sensor.
    ///
    /// A strongly-typed quiver column: guaranteed to be `Utf8` with no nulls.
    sensor: Column<Utf8>,

    /// Measured temperature.
    ///
    /// `Column<Option<f64>>`: the *values* may be null.
    /// (`Option<Column<f64>>` would instead mean the whole *column* may be missing.)
    temperature: Column<Option<f64>>,

    /// A raw arrow array: any datatype, any nullability.
    ///
    /// Use raw arrow types when you *want* things to be dynamic.
    comment: ArrayRef,
}

fn main() -> Result<(), Box<dyn std::error::Error>> {
    // Build a typed record — the struct literal is the builder:
    let measurements = Measurements {
        metadata: BTreeMap::from([("origin".to_owned(), "lab".to_owned())]),
        sensor: Column::from_values(["kitchen", "bedroom", "attic"]),
        temperature: Column::from_values([Some(22.1), None, Some(30.5)]),
        comment: Arc::new(StringArray::from(vec!["cozy", "quiet", "spooky"])),
    };

    // Convert into an ordinary arrow `RecordBatch` (fails on column length mismatch):
    let batch = measurements.into_record_batch()?;

    // … write to disk, send over the network, etc …

    // Or extract just a single, strongly-typed column:
    let temperatures = Measurements::COLUMN_TEMPERATURE.extract(&batch)?;
    println!(
        "max: {:?} °C",
        temperatures.iter().flatten().max_by(f64::total_cmp)
    );

    // Parse it back: validates the schema, then downcasts the columns (zero-copy):
    let measurements = Measurements::from_record_batch(batch)?;

    println!("origin: {:?}", measurements.metadata.get("origin"));

    // Typed columns iterate without any downcasting or unwrapping:
    for (sensor, temperature) in
        std::iter::zip(measurements.sensor.iter(), measurements.temperature.iter())
    {
        match temperature {
            Some(temperature) => println!("{sensor}: {temperature} °C"),
            None => println!("{sensor}: no reading"),
        }
    }

    // The raw arrow column is dynamically typed:
    println!("comments: {:?}", measurements.comment);

    Ok(())
}