zero4rs 2.0.0

zero4rs is a powerful, pragmatic, and extremely fast web framework for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

use csv::{ReaderBuilder, Terminator};
use std::collections::HashMap;

type StringMap = Vec<HashMap<String, Option<String>>>;

pub fn read_file<T>(
    csv_file: &str,
    delimiter: char,
    has_header: bool,
) -> std::io::Result<(Vec<T>, usize)>
where
    T: serde::de::DeserializeOwned,
{
    let mut rdr = csv::ReaderBuilder::new()
        .has_headers(has_header)
        .delimiter(delimiter as u8)
        .double_quote(true)
        .escape(Some(b'\\'))
        .flexible(true) // 自适应
        // .comment(Some(b'#'))
        .from_path(csv_file)?;

    let headers = if has_header {
        Some(rdr.byte_headers()?.clone())
    } else {
        None
    };

    let mut count = 0;
    let mut list: Vec<T> = vec![];
    let mut raw_record = csv::ByteRecord::new();

    while rdr.read_byte_record(&mut raw_record)? {
        log::info!("raw_record: {:?}", raw_record);

        let result = match headers {
            Some(ref headers) => raw_record.deserialize::<T>(Some(headers)),
            None => raw_record.deserialize::<T>(None),
        };

        match result {
            Ok(record) => {
                list.push(record);
                count += 1;
            }
            Err(e) => {
                count += 1;
                log::error!("read_csv_error: error={:?}", e);
            }
        }
    }

    Ok((list, count as usize))
}

// csv::read_csv(
//     &csv_file,
//     vec![
//         "colume1", "colume2", "colume3", "colume4", "colume5", "colume6",
//     ],
//     10000,
//     &|map_list| {
//         println!("read_csv-handler: current_map={:?}", map_list.len());
//     },
// )?;
// https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/second-edition/ch08-03-hash-maps.html
pub fn read_csv(
    file_path: &str,
    fields: Vec<&str>,
    batch_size: usize,
    handler: &dyn Fn(&StringMap),
) -> Result<(), anyhow::Error> {
    let mut builder = ReaderBuilder::new();

    builder
        .double_quote(false)
        .comment(Some(b'-'))
        .has_headers(false)
        .terminator(Terminator::CRLF);

    let mut reader = builder.from_path(file_path)?;

    let mut map_list = vec![];
    let mut total_count = 0;

    for line in reader.records().flatten() {
        let mut curr_row = HashMap::new();

        for (i, field) in fields.iter().enumerate() {
            if let Some(cell) = line.get(i) {
                if cell == "NULL" {
                    curr_row.insert(field.to_string(), None);
                } else {
                    curr_row.insert(field.to_string(), Some(cell.to_string()));
                }
            } else {
                curr_row.insert(field.to_string(), None);
            }
        }

        map_list.push(curr_row);
        total_count += 1;

        if total_count % batch_size == 0 {
            handler(&map_list);
            map_list.clear();
        }
    }

    handler(&map_list);

    log::info!(
        "read_csv: total_count={}, map_size={}",
        total_count,
        map_list.len()
    );

    Ok(())
}