Skip to main content

test_pipeline/
test_pipeline.rs

1use datacortex_core::dcx::Mode;
2use datacortex_core::format::{detect_format, preprocess, reverse_preprocess};
3
4fn main() {
5    let data = std::fs::read("corpus/test-ndjson.ndjson").unwrap();
6    let format = detect_format(&data);
7    eprintln!("Format: {:?}", format);
8    eprintln!("Original: {} bytes", data.len());
9
10    let (preprocessed, chain) = preprocess(&data, format, Mode::Balanced);
11    eprintln!("Preprocessed: {} bytes", preprocessed.len());
12    eprintln!("Transforms: {}", chain.records.len());
13    for (i, rec) in chain.records.iter().enumerate() {
14        eprintln!(
15            "  [{}] ID={} metadata={} bytes",
16            i,
17            rec.id,
18            rec.metadata.len()
19        );
20    }
21
22    let restored = reverse_preprocess(&preprocessed, &chain);
23    if restored == data {
24        eprintln!("Roundtrip: OK");
25    } else {
26        eprintln!("Roundtrip: FAILED ({} vs {})", restored.len(), data.len());
27    }
28}