pub fn preprocess(
data: &[u8],
format: FormatHint,
mode: Mode,
) -> (Vec<u8>, TransformChain)Expand description
Apply format-aware preprocessing transforms. Returns (preprocessed_data, transform_chain).
NDJSON columnar: ALL modes (grouping similar values helps both zstd and CM). Key interning: Balanced/Max only (hurts Fast mode due to zstd redundancy). For NDJSON, columnar is applied FIRST — if it succeeds, key interning is skipped (keys are already removed from the data stream by the columnar transform).
Examples found in repository?
examples/test_pipeline.rs (line 10)
4fn main() {
5 let data = std::fs::read("corpus/test-ndjson.ndjson").unwrap();
6 let format = detect_format(&data);
7 eprintln!("Format: {:?}", format);
8 eprintln!("Original: {} bytes", data.len());
9
10 let (preprocessed, chain) = preprocess(&data, format, Mode::Balanced);
11 eprintln!("Preprocessed: {} bytes", preprocessed.len());
12 eprintln!("Transforms: {}", chain.records.len());
13 for (i, rec) in chain.records.iter().enumerate() {
14 eprintln!(
15 " [{}] ID={} metadata={} bytes",
16 i,
17 rec.id,
18 rec.metadata.len()
19 );
20 }
21
22 let restored = reverse_preprocess(&preprocessed, &chain);
23 if restored == data {
24 eprintln!("Roundtrip: OK");
25 } else {
26 eprintln!("Roundtrip: FAILED ({} vs {})", restored.len(), data.len());
27 }
28}More examples
examples/dump_transform.rs (line 31)
14fn main() {
15 let args: Vec<String> = env::args().collect();
16 if args.len() != 3 {
17 eprintln!("Usage: {} <input> <output>", args[0]);
18 process::exit(1);
19 }
20
21 let input_path = &args[1];
22 let output_path = &args[2];
23
24 let data = fs::read(input_path).expect("Failed to read input file");
25
26 // Detect format (try extension first, then content).
27 let format = datacortex_core::format::detect_from_extension(input_path)
28 .unwrap_or_else(|| detect_format(&data));
29
30 // Run preprocessing.
31 let (transformed, chain) = preprocess(&data, format, Mode::Balanced);
32
33 // Calculate metadata size.
34 let metadata_bytes = chain.serialize();
35 let metadata_size = metadata_bytes.len();
36
37 // Write transformed data to output.
38 fs::write(output_path, &transformed).expect("Failed to write output file");
39
40 // Report stats.
41 let name = std::path::Path::new(input_path)
42 .file_name()
43 .unwrap()
44 .to_str()
45 .unwrap();
46
47 eprintln!("=== {} ===", name);
48 eprintln!("Format: {:?}", format);
49 eprintln!("Original: {} bytes", data.len());
50 eprintln!("Transformed: {} bytes", transformed.len());
51 eprintln!("Metadata: {} bytes", metadata_size);
52 eprintln!(
53 "Total (transform + meta): {} bytes",
54 transformed.len() + metadata_size
55 );
56 eprintln!(
57 "Transform ratio: {:.1}%",
58 (transformed.len() + metadata_size) as f64 / data.len() as f64 * 100.0
59 );
60
61 if transformed.len() == data.len() {
62 eprintln!("NOTE: No transform applied (output == input)");
63 }
64}