csv_lib/
lib.rs

1//! # Csv_lib Crate
2//!
3//! A Rust library to read/write CSV files in the fastest way I know.
4//! 
5//! For further information, and complete docs, you can check the repo [here](https://github.com/PTechSoftware/csv_lib)
6//!
7//! ## 3rd Party Crates Used:
8//!
9//! | Crate | Link |
10//! | :---- | :---- |
11//! | Memmap2 | [memmap2 crate](https://docs.rs/memmap2/latest/memmap2/) |
12//! | Memchr | [memchr crate](https://docs.rs/memchr/latest/memchr/) |
13//! | num_cpus | [num_cpus crate](https://docs.rs/memchr/latest/memchr/) |
14//!
15//! ## Features
16//! - Zero-copy parsing
17//! - Custom delimiters support
18//! - Escape string support
19//! - Direct mapping from memory
20//! - Multicore Process
21//! - Low Ram Usage, even on big files
22//!
23//!
24//! ## Performance
25//! This library is designed to process large CSV files.  
26//! Successfully tested on a 1 billion lines CSV file.
27//! To test performance, run in `release` config, it improves a lot the performance
28//!
29//! ## Contact
30//! If you have any questions, contact me on [LinkedIn](https://www.linkedin.com/in/ignacio-p%C3%A9rez-panizza-322844165/)
31
32
33extern crate core;
34
35pub mod models;
36pub mod io;
37pub mod helpers;
38pub mod csv;
39pub mod extensions;
40pub mod features;
41pub mod decoders;
42pub mod encoders;
43pub mod parallel;
44pub mod macros;
45
46#[cfg(test)]
47mod test {
48    use std::collections::HashSet;
49    use std::sync::{Arc, Mutex};
50    use crate::csv::csv_reader::CsvReaderWithMap;
51    use crate::decoders::decoders::Encoding;
52    use crate::{get_bool, get_f64, get_str};
53    use crate::models::csv_config::CsvConfig;
54    use crate::models::shared::Shared;
55    use crate::parallel::parallel_reader::parallel_processing_csv;
56    use crate::parallel::row_parallel::RowParallel;
57
58    #[test]
59    fn read_csv_one_core(){
60        //Create Config
61        let cfg = CsvConfig::new(
62            b',',
63            0u8,
64            b'\n',
65            Encoding::Windows1252,
66            false
67        );
68        //Open the file
69        let mut f = match CsvReaderWithMap::open("data.csv", &cfg) {
70            Ok(f) => f,
71            Err(e) => panic!("{}", e)
72        };
73        // We extract different' s country's of the dataset :
74        // For example:
75        //Create a Hash Acumulator
76        let mut cities :HashSet<String>= HashSet::with_capacity(195);
77        //Iter over rows
78        while let Some(mut row) = f.next_raw() {
79            //Extract Field index 6 starting on 0
80            let city = row.get_index(6 );
81            // Decode bytes as &str
82            let name = city.get_utf8_as_str();
83            let _ = get_bool!(row,1);
84            let num = city.get_i8();
85            //Check and accumulate
86            if !cities.contains(name){
87                cities.insert(name.to_string());
88            }
89        }
90        assert_ne!(cities.len(), 0);
91    }
92    #[test]
93    fn read_csv_multicore(){
94        //Create Config
95        let cfg = CsvConfig::new(
96            b',',
97            0u8,
98            b'\n',
99            Encoding::Windows1252,
100            false
101        );
102        //Open the file
103        let f = match CsvReaderWithMap::open("data.csv", &cfg) {
104            Ok(f) => f,
105            Err(e) => panic!("{}", e)
106        };
107
108        //Get Slice Reference
109        let data = f.get_slice();
110        //Create a shared counter
111        let shared = Shared::<i32>::default();
112        //Create de clousere executed on each thread (the ARC Mutex type must be the same as Shared)
113        let closure = |row: &mut RowParallel<'_>, id_thread:usize, target: Arc<Mutex<i32>>| {
114            //Get thread Id
115            let _ = id_thread;
116            //Access actual row
117            let _actual = row.get_row();
118            //Peek nex row
119            let next = row.peek_next();
120            //Do some stuff
121            // ...
122            
123            let mut lock = target.lock().unwrap();
124            *lock += 1;
125        };
126        //Execute parallel process
127        parallel_processing_csv(
128            data,
129            b'\n',
130            b';',
131            b'"',
132            false,
133            closure,
134            shared.arc(),
135        );
136        println!("Iterated Lines: {:.2}", shared.lock())
137    }
138}
csv_lib/lib.rs

csv_lib/
lib.rs
