uiuifree-text-data 0.1.4

use std::error::Error;
use std::sync::Mutex;

use rayon::prelude::*;
use uiuifree_normalize::{free_text, remove_html};

pub struct CsvTokenizer {
    path: String,
    // callable_test: fn(String) -> String,
}

impl CsvTokenizer {
    pub fn new(path: &str) -> CsvTokenizer {
        CsvTokenizer {
            path: path.to_string()
            // callable_test: |x| {
            //     x
            // },
        }
    }
    pub fn trim_csv(input: &str, output: &str) {
        // 読み込むファイルパスの設定
        match CsvTokenizer::new(input).execute(output) {
            Ok(_) => {}
            Err(e) => {
                println!("{}", e)
            }
        };
    }

    // fn set_fn(&mut self, callable_test: fn(String) -> String) {
    //     self.callable_test = callable_test
    // }

    pub fn headers(&self) -> Result<Vec<String>, Box<dyn Error>> {
        let path = String::from(self.path.as_str());
        let mut rdr = csv::Reader::from_path(path)?;

        let mut res = Vec::new();
        for i in rdr.headers()?.iter() {
            res.push(i.to_string());
        }
        Ok(res)
    }
    pub fn records(&self) -> Result<Vec<Vec<String>>, Box<dyn Error>> {
        let path = String::from(self.path.as_str());
        let mut rdr = csv::Reader::from_path(path)?;

        let mut rows: Vec<Vec<String>> = Vec::new();
        for result in rdr.records() {
            let record = result?;
            let mut row: Vec<String> = Vec::new();
            for s in record.iter() {
                row.push(s.to_string());
            }
            rows.push(row);
        }
        Ok(rows)
    }

    pub async fn process<F, T, U>(&mut self, callable: F) -> Result<Vec<U>, Box<dyn Error>>
        where
            F: Fn(Vec<String>) -> T,
            T: std::future::Future<Output=U> + std::marker::Send + 'static,
            U: std::marker::Send + 'static
    {
        let path = String::from(self.path.as_str());
        let mut rdr = csv::Reader::from_path(path)?;

        let mut handles = vec![];
        let mut response = vec![];

        for result in rdr.records() {
            let record = result?;
            let mut row: Vec<String> = Vec::new();
            for s in record.iter() {
                row.push(free_text(s.to_string()));
                // row.push(remove_html(free_text(s.to_string())))
                // row.push(remove_html(free_text(s.to_string())))
            }
            handles.push(tokio::spawn(callable(row)));
        }
        for handle in handles {
            response.push(handle.await.unwrap())
        }

        return Ok(response);
    }
    pub fn execute(&self, out_path: &str) -> Result<(), Box<dyn Error>> {
        let path = String::from(self.path.as_str());
        // Build the CSV reader and iterate over each record.
        let mut rdr = csv::Reader::from_path(path)?;
        let mut w = csv::Writer::from_path(out_path)?;
        let headers = rdr.headers()?;

        w.write_record(headers)?;
        let mut rows: Vec<Vec<String>> = Vec::new();

        for record in rdr.records() {
            if record.is_err() {
                break;
            }
            let record = record.unwrap();
            let mut row: Vec<String> = Vec::new();
            for s in record.iter() {
                row.push(s.to_string());
            }
            rows.push(row);
        }

        let vec: Mutex<Vec<Vec<String>>> = Mutex::new(Vec::new());
        rows.par_iter_mut().for_each(|row| {
            let mut new_rows = Vec::new();
            for v in row {
                new_rows.push(remove_html(free_text(v.to_string())))
                // HW 落ちるので書き換え
                // new_rows.push(remove_html(space(v.to_string())))
            }
            vec.lock().unwrap().push(new_rows);
        });
        let rows = vec.lock().unwrap().to_vec();
        for row in rows {
            w.write_record(row)?;
        }
        Ok(())
    }
}