1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

use std::error::Error;
use std::sync::Mutex;

use rayon::prelude::*;
use uiuifree_normalize::{free_text, remove_html};

pub struct CsvTokenizer {
    path: String,
    // callable_test: fn(String) -> String,
}

impl CsvTokenizer {
    pub fn new(path: &str) -> CsvTokenizer {
        CsvTokenizer {
            path: path.to_string()
            // callable_test: |x| {
            //     x
            // },
        }
    }
    pub fn trim_csv(input: &str, output: &str) {
        // 読み込むファイルパスの設定
        match CsvTokenizer::new(input).execute(output) {
            Ok(_) => {}
            Err(e) => {
                println!("{}", e)
            }
        };
    }

    // fn set_fn(&mut self, callable_test: fn(String) -> String) {
    //     self.callable_test = callable_test
    // }

    pub fn headers(&self) -> Result<Vec<String>, Box<dyn Error>> {
        let path = String::from(self.path.as_str());
        let mut rdr = csv::Reader::from_path(path)?;

        let mut res = Vec::new();
        for i in rdr.headers()?.iter() {
            res.push(i.to_string());
        }
        Ok(res)
    }
    pub fn records(&self) -> Result<Vec<Vec<String>>, Box<dyn Error>> {
        let path = String::from(self.path.as_str());
        let mut rdr = csv::Reader::from_path(path)?;

        let mut rows: Vec<Vec<String>> = Vec::new();
        for result in rdr.records() {
            let record = result?;
            let mut row: Vec<String> = Vec::new();
            for s in record.iter() {
                row.push(s.to_string());
            }
            rows.push(row);
        }
        Ok(rows)
    }

    pub async fn process<F, T, U>(&mut self, callable: F) -> Result<Vec<U>, Box<dyn Error>>
        where
            F: Fn(Vec<String>) -> T,
            T: std::future::Future<Output=U> + std::marker::Send + 'static,
            U: std::marker::Send + 'static
    {
        let path = String::from(self.path.as_str());
        let mut rdr = csv::Reader::from_path(path)?;

        let mut handles = vec![];
        let mut response = vec![];

        for result in rdr.records() {
            let record = result?;
            let mut row: Vec<String> = Vec::new();
            for s in record.iter() {
                row.push(free_text(s.to_string()));
                // row.push(remove_html(free_text(s.to_string())))
                // row.push(remove_html(free_text(s.to_string())))
            }
            handles.push(tokio::spawn(callable(row)));
        }
        for handle in handles {
            response.push(handle.await.unwrap())
        }

        return Ok(response);
    }
    pub fn execute(&self, out_path: &str) -> Result<(), Box<dyn Error>> {
        let path = String::from(self.path.as_str());
        // Build the CSV reader and iterate over each record.
        let mut rdr = csv::Reader::from_path(path)?;
        let mut w = csv::Writer::from_path(out_path)?;
        let headers = rdr.headers()?;

        w.write_record(headers)?;
        let mut rows: Vec<Vec<String>> = Vec::new();

        for record in rdr.records() {
            if record.is_err() {
                break;
            }
            let record = record.unwrap();
            let mut row: Vec<String> = Vec::new();
            for s in record.iter() {
                row.push(s.to_string());
            }
            rows.push(row);
        }

        let vec: Mutex<Vec<Vec<String>>> = Mutex::new(Vec::new());
        rows.par_iter_mut().for_each(|row| {
            let mut new_rows = Vec::new();
            for v in row {
                new_rows.push(remove_html(free_text(v.to_string())))
                // HW 落ちるので書き換え
                // new_rows.push(remove_html(space(v.to_string())))
            }
            vec.lock().unwrap().push(new_rows);
        });
        let rows = vec.lock().unwrap().to_vec();
        for row in rows {
            w.write_record(row)?;
        }
        Ok(())
    }
}