reddit-search 0.8.3

A search tool for the pushshift.io Reddit dumps.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

use rayon::prelude::*;

pub(crate) fn process_line(line: &str, search_strings: &Vec<String>) -> Option<String> {
    // switched this away from serde_json because it was very slow, and we don't need to parse the whole line
    if search_strings.iter().any(|formats| { line.to_lowercase().contains(&formats.to_lowercase()) }) {
        Some(line.to_string())
    } else {
        None
    }
}

pub(crate) fn process_chunk(lines: Vec<String>, search_strings: &Vec<String>) -> Vec<String> {
    lines.into_par_iter()
        .filter_map(|line| process_line(&line, search_strings))
        .collect()
}