datui-lib 0.4.5

Data Exploration in the Terminal (library)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

use super::*;
use object_store::{ObjectStore, ObjectStoreExt, PutPayload, path::Path as OsPath};
use polars::prelude::*;
use std::sync::Arc;

/// One row of Parquet with the given columns.
fn parquet(columns: &[&str]) -> Vec<u8> {
    let mut frame = DataFrame::new(
        1,
        columns
            .iter()
            .map(|c| Column::new((*c).into(), &[1i32]))
            .collect::<Vec<_>>(),
    )
    .unwrap();
    let mut bytes = Vec::new();
    ParquetWriter::new(&mut bytes).finish(&mut frame).unwrap();
    bytes
}

/// Put `files` in a store and ask what the directory is.
fn kind_of(files: &[(&str, &[&str])]) -> Option<crate::home::discover::EntryKind> {
    let rt = tokio::runtime::Runtime::new().unwrap();
    let store: Arc<dyn ObjectStore> = Arc::new(object_store::memory::InMemory::new());
    rt.block_on(async {
        let mut objects = Vec::new();
        for (key, columns) in files {
            let bytes = parquet(columns);
            objects.push((format!("data/{key}"), bytes.len() as u64));
            store
                .put(
                    &OsPath::from(format!("data/{key}")),
                    PutPayload::from(bytes),
                )
                .await
                .unwrap();
        }
        kind_from_footers(&store, &objects).await
    })
}

/// The shape that prompted this, as it sits in a bucket: one file per table.
#[test]
fn separate_tables_in_a_bucket_are_a_directory() {
    let kind = kind_of(&[
        ("circuits.parquet", &["circuit_id", "lat", "lng"]),
        ("drivers.parquet", &["driver_id", "code", "nationality"]),
        ("laps.parquet", &["lap", "position", "time_millis"]),
    ]);
    assert_eq!(kind, Some(crate::home::discover::EntryKind::Directory));
}

#[test]
fn parts_of_one_table_stay_one_dataset() {
    let kind = kind_of(&[
        ("part-00000.parquet", &["id", "ts", "amount"]),
        ("part-00001.parquet", &["id", "ts", "amount"]),
        ("part-00002.parquet", &["id", "ts", "amount"]),
    ]);
    assert_eq!(kind, Some(crate::home::discover::EntryKind::MultiFile));
}

/// A dataset that gained columns over the years is still one dataset, and the
/// files read are its ends, which is where the difference is.
#[test]
fn a_dataset_that_gained_columns_stays_one_dataset() {
    let kind = kind_of(&[
        ("date=2009-01-03.parquet", &["id", "ts"]),
        ("date=2015-06-01.parquet", &["id", "ts", "fee"]),
        (
            "date=2025-06-01.parquet",
            &["id", "ts", "fee", "witness", "address"],
        ),
    ]);
    assert_eq!(kind, Some(crate::home::discover::EntryKind::MultiFile));
}

/// Nothing readable means nothing decided, and the listing's answer stands.
#[test]
fn a_directory_that_cannot_be_read_is_left_as_it_was() {
    let rt = tokio::runtime::Runtime::new().unwrap();
    let store: Arc<dyn ObjectStore> = Arc::new(object_store::memory::InMemory::new());
    let objects = vec![
        ("data/a.parquet".to_string(), 10),
        ("data/b.parquet".to_string(), 10),
    ];
    assert_eq!(rt.block_on(kind_from_footers(&store, &objects)), None);
}

/// One file cannot disagree with anything.
#[test]
fn a_single_file_decides_nothing() {
    assert_eq!(kind_of(&[("only.parquet", &["a", "b"])]), None);
}

/// The ends of a listing are where a table-per-file directory differs; its head can
/// be three files of the same table by alphabetical accident.
#[test]
fn the_files_read_span_the_listing() {
    use crate::home::discover::spread;
    assert_eq!(spread(1), vec![0]);
    assert_eq!(spread(2), vec![0, 1]);
    assert_eq!(spread(3), vec![0, 1, 2]);
    assert_eq!(spread(15), vec![0, 7, 14]);
}