Skip to main content

parse_corpus_parallel

Function parse_corpus_parallel 

Source
pub fn parse_corpus_parallel<T, F>(
    sources: &[(Vec<u8>, Vec<u8>)],
    cache: &ParsedSourceLru<T>,
    parse: F,
) -> Vec<Arc<T>>
where T: Send + Sync, F: Fn(&[u8]) -> T + Sync,
Expand description

Parse every (content, extra_key) pair in sources in parallel, memoising through cache. Returns Arc<T> values in input order.

§Type parameters

  • T - parsed artifact; must be Send + Sync so it can cross thread boundaries inside Arc<T>.
  • F - parse function; must be Sync because it is called from multiple rayon workers concurrently.

§Example

use vyre_libs::parsing::source_cache::ParsedSourceLru;
use vyre_libs::parsing::parallel_parse::parse_corpus_parallel;
use std::sync::Arc;

let cache = ParsedSourceLru::with_capacity(64);
let sources: Vec<(Vec<u8>, Vec<u8>)> = vec![
    (b"int main() {}".to_vec(), b"".to_vec()),
    (b"float x;".to_vec(), b"-O2".to_vec()),
];
let results: Vec<Arc<usize>> = parse_corpus_parallel(&sources, &cache, |src| src.len());
assert_eq!(*results[0], 13);
assert_eq!(*results[1], 8);