use crate::catcode::CatTable;
use crate::lexer::Lexer;
use crate::token::Token;
pub struct Chunk {
pub start: usize,
pub tokens: Vec<Token>,
}
static CONFIGURED: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
fn requested(jobs: Option<usize>) -> Option<usize> {
jobs.or_else(|| {
std::env::var("TEXRS_THREADS")
.ok()
.and_then(|v| v.trim().parse::<usize>().ok())
})
.filter(|n| *n >= 1)
}
pub fn configure(jobs: Option<usize>) {
let Some(n) = requested(jobs) else { return };
if CONFIGURED.set(n).is_ok() {
let _ = rayon::ThreadPoolBuilder::new()
.num_threads(n)
.build_global();
}
}
pub fn default_threads() -> usize {
CONFIGURED.get().copied().unwrap_or_else(|| {
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
})
}
const ORDERED_BYTES: usize = 64 * 1024;
pub fn map_ordered<I: Sync, T: Send>(
items: &[I],
bytes: usize,
f: impl Fn(&I) -> T + Sync + Send,
) -> Vec<T> {
use rayon::prelude::*;
match bytes >= ORDERED_BYTES {
true => items.par_iter().map(f).collect(),
false => items.iter().map(f).collect(),
}
}
pub fn line_aligned_bounds(src: &str, n: usize) -> Vec<usize> {
if n <= 1 || src.is_empty() {
return vec![0];
}
let target = src.len() / n;
if target == 0 {
return vec![0];
}
let bytes = src.as_bytes();
let mut bounds = vec![0usize];
let mut want = target;
while want < src.len() {
match bytes[want..].iter().position(|b| *b == b'\n') {
Some(off) => {
let at = want + off + 1;
if at >= src.len() {
break;
}
if at > *bounds.last().expect("seeded") {
bounds.push(at);
}
want = at + target;
}
None => break,
}
}
bounds
}
pub fn lex_parallel(src: &str, cats: &CatTable, threads: usize) -> Vec<Chunk> {
let bounds = line_aligned_bounds(src, threads.max(1));
if bounds.len() == 1 {
return vec![Chunk {
start: 0,
tokens: lex_range(src, cats),
}];
}
let mut out: Vec<Chunk> = Vec::with_capacity(bounds.len());
std::thread::scope(|s| {
let mut handles = Vec::with_capacity(bounds.len());
for (i, &start) in bounds.iter().enumerate() {
let end = bounds.get(i + 1).copied().unwrap_or(src.len());
let piece = &src[start..end];
handles.push((start, s.spawn(move || lex_range(piece, cats))));
}
for (start, h) in handles {
let tokens = h.join().unwrap_or_default();
out.push(Chunk { start, tokens });
}
});
out
}
fn lex_range(piece: &str, cats: &CatTable) -> Vec<Token> {
let mut lx = Lexer::new(piece);
let mut toks = Vec::new();
while let Some(t) = lx.next_token(cats) {
toks.push(t);
}
toks
}
pub fn lex_parallel_flat(src: &str, cats: &CatTable, threads: usize) -> Vec<Token> {
lex_parallel(src, cats, threads)
.into_iter()
.flat_map(|c| c.tokens)
.collect()
}
pub type Placed = (Token, usize);
fn line_starts(chars: &[char], want: usize) -> Vec<usize> {
if want <= 1 || chars.is_empty() {
return vec![0];
}
let target = chars.len() / want;
if target == 0 {
return vec![0];
}
let mut bounds = vec![0usize];
let mut i = target;
while i < chars.len() {
match chars[i..].iter().position(|c| *c == '\n') {
Some(off) => {
let at = i + off + 1;
if at >= chars.len() {
break;
}
if at > *bounds.last().expect("seeded") {
bounds.push(at);
}
i = at + target;
}
None => break,
}
}
bounds
}
pub static DROP_GEN: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
pub static DROP_EXHAUST: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
pub static PRELEX_CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
pub static PRELEX_CHARS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
pub fn prelex(chars: &[char], from: usize, cats: &CatTable, threads: usize) -> Vec<Placed> {
PRELEX_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
PRELEX_CHARS.fetch_add(
chars.len().saturating_sub(from),
std::sync::atomic::Ordering::Relaxed,
);
let rest = &chars[from.min(chars.len())..];
if rest.is_empty() {
return Vec::new();
}
let bounds = line_starts(rest, threads.max(1));
if bounds.len() == 1 {
return lex_slice(rest, from, cats).0;
}
let mut parts: Vec<(Vec<Placed>, bool)> = Vec::with_capacity(bounds.len());
std::thread::scope(|s| {
let mut hs = Vec::with_capacity(bounds.len());
for (i, &start) in bounds.iter().enumerate() {
let end = bounds.get(i + 1).copied().unwrap_or(rest.len());
let piece = &rest[start..end];
let base = from + start;
hs.push(s.spawn(move || lex_slice(piece, base, cats)));
}
for h in hs {
parts.push(h.join().unwrap_or_default());
}
});
let mut out = Vec::new();
for (toks, stopped) in parts {
out.extend(toks);
if stopped {
break;
}
}
out
}
fn lex_slice(piece: &[char], base: usize, cats: &CatTable) -> (Vec<Placed>, bool) {
let mut lx = Lexer::from_chars(piece.to_vec());
let mut out = Vec::new();
while let Some(t) = lx.next_token(cats) {
if !lx.take_invalid().is_empty() {
return (out, true);
}
out.push((t, base + lx.pos()));
}
let stopped = !lx.take_invalid().is_empty();
(out, stopped)
}