datu 0.3.4

datu - a data file utility
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

use super::DataframeSelect;
use super::DataframeTail;
use crate::FileType;
use crate::pipeline::DataframeParquetReader;
use crate::pipeline::DataframeToRecordBatch;
use crate::pipeline::RecordBatchAvroWriter;
use crate::pipeline::Step;
use crate::pipeline::csv::DataframeCsvWriter;
use crate::pipeline::read::ReadArgs;
use crate::pipeline::write::WriteArgs;

fn temp_path(dir: &tempfile::TempDir, name: &str) -> String {
    dir.path()
        .join(name)
        .to_str()
        .expect("Failed to convert path to string")
        .to_string()
}

#[tokio::test(flavor = "multi_thread")]
async fn test_dataframe_steps_parquet_tail_to_csv() {
    let read_args = ReadArgs::new("fixtures/table.parquet", FileType::Parquet);
    let source = DataframeParquetReader { args: read_args }
        .execute(())
        .await
        .unwrap();
    let source = DataframeSelect { select: None }
        .execute(source)
        .await
        .unwrap();
    let source = DataframeTail {
        input_path: "fixtures/table.parquet".to_string(),
        input_file_type: FileType::Parquet,
        n: 2,
    }
    .execute(source)
    .await
    .unwrap();
    let temp_dir = tempfile::tempdir().unwrap();
    let output = temp_path(&temp_dir, "out.csv");
    let write_args = WriteArgs {
        path: output.clone(),
        file_type: FileType::Csv,
        sparse: None,
        pretty: None,
    };
    DataframeCsvWriter { args: write_args }
        .execute(Box::new(source))
        .await
        .unwrap();
    assert!(std::path::Path::new(&output).exists());
}

#[tokio::test(flavor = "multi_thread")]
async fn test_dataframe_to_record_batch_record_batch_avro_writer() {
    let read_args = ReadArgs::new("fixtures/table.parquet", FileType::Parquet);
    let source = DataframeParquetReader { args: read_args }
        .execute(())
        .await
        .unwrap();
    let reader = DataframeToRecordBatch::try_new(source).await.unwrap();
    let temp_dir = tempfile::tempdir().unwrap();
    let output = temp_path(&temp_dir, "out.avro");
    let write_args = WriteArgs {
        path: output.clone(),
        file_type: FileType::Avro,
        sparse: None,
        pretty: None,
    };
    RecordBatchAvroWriter { args: write_args }
        .execute(Box::new(reader))
        .await
        .unwrap();
    assert!(std::path::Path::new(&output).exists());
}