1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
use std::path::PathBuf; use rayon::iter::{IntoParallelIterator, ParallelIterator}; mod chunks; mod counter; mod key_finder; pub use chunks::chunks; pub use counter::{Counter, KeyCount}; pub use key_finder::KeyFinder; pub fn top_few_from_stream( path: PathBuf, kf: &KeyFinder, num: usize, ) -> anyhow::Result<Vec<KeyCount>> { let total = chunks(path)? .collect::<Vec<_>>() .into_par_iter() .map(|reader| { let mut counter = Counter::new(None); let mut s = String::new(); for ln in reader { s.clear(); if let Ok(key) = kf.key(&ln, &mut s) { counter.add(key, 1) } } counter }) .reduce(|| Counter::new(Some(num)), |l, r| l.merge(r)); Ok(total.top()) }