use std::any::Any;
use std::collections::VecDeque;
use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Condvar, Mutex};
use std::thread::JoinHandle;
use crate::coder_scratch::CoderScratch;
struct Completion {
remaining: AtomicUsize,
panic: Mutex<Option<Box<dyn Any + Send>>>,
}
impl Completion {
fn finish(&self, panic: Option<Box<dyn Any + Send>>, shared: &Shared) {
if let Some(payload) = panic {
let mut first_panic = self.panic.lock().unwrap();
if first_panic.is_none() {
*first_panic = Some(payload);
}
}
let _queue = shared.queue.lock().unwrap();
if self.remaining.fetch_sub(1, Ordering::Release) == 1 {
shared.activity.notify_all();
}
}
}
struct Task {
run: Box<dyn FnOnce(&mut CoderScratch) + Send + 'static>,
completion: Arc<Completion>,
}
impl Task {
fn execute(self, shared: &Shared, scratch: &mut CoderScratch) {
let Self { run, completion } = self;
let result = catch_unwind(AssertUnwindSafe(|| run(scratch)));
completion.finish(result.err(), shared);
}
}
struct Shared {
queue: Mutex<VecDeque<Task>>,
activity: Condvar,
shutdown: AtomicBool,
}
impl Shared {
fn push(&self, task: Task) {
self.queue.lock().unwrap().push_back(task);
self.activity.notify_one();
}
fn run_one(&self, scratch: &mut CoderScratch) -> bool {
let task = self.queue.lock().unwrap().pop_front();
if let Some(task) = task {
task.execute(self, scratch);
true
} else {
false
}
}
fn wait_for_activity(&self, remaining: &AtomicUsize) {
let queue = self.queue.lock().unwrap();
if queue.is_empty()
&& remaining.load(Ordering::Acquire) != 0
&& !self.shutdown.load(Ordering::Acquire)
{
let _queue = self.activity.wait(queue).unwrap();
}
}
}
pub(crate) struct ThreadPool {
shared: Arc<Shared>,
workers: Vec<JoinHandle<()>>,
num_threads: usize,
}
impl ThreadPool {
pub(crate) fn new(num_threads: usize) -> Self {
Self::new_with_scratch(num_threads, CoderScratch::default, None)
}
pub(crate) fn new_lossless(num_threads: usize) -> Self {
Self::new_with_scratch(num_threads, CoderScratch::lossless, None)
}
fn new_with_scratch(
num_threads: usize,
make_scratch: fn() -> CoderScratch,
stack_size: Option<usize>,
) -> Self {
let num_threads = num_threads.max(1);
let shared = Arc::new(Shared {
queue: Mutex::new(VecDeque::new()),
activity: Condvar::new(),
shutdown: AtomicBool::new(false),
});
let workers = (1..num_threads)
.map(|i| {
let shared = Arc::clone(&shared);
let mut builder = std::thread::Builder::new().name(format!("jixel-worker-{i}"));
if let Some(stack_size) = stack_size {
builder = builder.stack_size(stack_size);
}
builder
.spawn(move || {
let mut scratch = Box::new(make_scratch());
worker_loop(&shared, &mut scratch);
})
.expect("failed to start encoder worker")
})
.collect();
Self {
shared,
workers,
num_threads,
}
}
#[inline]
pub(crate) fn num_threads(&self) -> usize {
self.num_threads
}
pub(crate) fn map<T: Send>(&self, len: usize, f: impl Fn(usize) -> T + Sync) -> Vec<T> {
if self.num_threads <= 1 || len <= 1 {
return (0..len).map(f).collect();
}
let mut scratch = CoderScratch::lossless();
self.steal_map(&mut scratch, len, |i, _| f(i))
}
pub(crate) fn for_each_mut<T: Send>(&self, items: &mut [T], f: impl Fn(usize, &mut T) + Sync) {
if self.num_threads <= 1 || items.len() <= 1 {
for (i, item) in items.iter_mut().enumerate() {
f(i, item);
}
return;
}
let mut scratch = CoderScratch::lossless();
self.steal_for_each_mut(&mut scratch, items, |i, item, _| f(i, item));
}
pub(crate) fn join<A: Send, B: Send>(
&self,
first: impl FnOnce() -> A + Send,
second: impl FnOnce() -> B + Send,
) -> (A, B) {
let (mut first, mut second) = (Some(first), Some(second));
let (mut a, mut b) = (None, None);
let mut run_first = || a = Some(first.take().unwrap()());
let mut run_second = || b = Some(second.take().unwrap()());
let mut jobs: [&mut (dyn FnMut() + Send); 2] = [&mut run_first, &mut run_second];
self.for_each_mut(&mut jobs, |_, run| run());
(a.unwrap(), b.unwrap())
}
#[cfg(test)]
pub(crate) fn on_small_stack(f: impl FnOnce() + Send + 'static) {
let pool = Self::new_with_scratch(2, CoderScratch::lossless, Some(64 * 1024));
let completion = Arc::new(Completion {
remaining: AtomicUsize::new(1),
panic: Mutex::new(None),
});
pool.shared.push(Task {
run: Box::new(move |_| f()),
completion: Arc::clone(&completion),
});
let mut queue = pool.shared.queue.lock().unwrap();
while completion.remaining.load(Ordering::Acquire) != 0 {
queue = pool.shared.activity.wait(queue).unwrap();
}
drop(queue);
if let Some(payload) = completion.panic.lock().unwrap().take() {
resume_unwind(payload);
}
}
pub(crate) fn steal_map<T, F>(
&self,
caller_scratch: &mut CoderScratch,
len: usize,
f: F,
) -> Vec<T>
where
T: Send,
F: Fn(usize, &mut CoderScratch) -> T + Sync,
{
self.steal_map_with_threads(caller_scratch, len, self.num_threads, f)
}
pub(crate) fn steal_map_with_threads<T, F>(
&self,
caller_scratch: &mut CoderScratch,
len: usize,
max_threads: usize,
f: F,
) -> Vec<T>
where
T: Send,
F: Fn(usize, &mut CoderScratch) -> T + Sync,
{
self.steal_map_with_threads_shared(caller_scratch, len, max_threads, &f)
}
fn steal_map_with_threads_shared<T>(
&self,
caller_scratch: &mut CoderScratch,
len: usize,
max_threads: usize,
f: &(dyn Fn(usize, &mut CoderScratch) -> T + Sync),
) -> Vec<T>
where
T: Send,
{
let mut slots: Vec<Option<T>> = Vec::with_capacity(len);
slots.resize_with(len, || None);
let base = SyncPtr(slots.as_mut_ptr());
let write = move |i: usize, scratch: &mut CoderScratch| {
let value = f(i, scratch);
unsafe { *base.add(i) = Some(value) };
};
self.steal_indices(caller_scratch, len, max_threads, &write);
slots.into_iter().map(Option::unwrap).collect()
}
pub(crate) fn steal_for_each_mut<T, F>(
&self,
caller_scratch: &mut CoderScratch,
items: &mut [T],
f: F,
) where
T: Send,
F: Fn(usize, &mut T, &mut CoderScratch) + Sync,
{
self.steal_for_each_mut_with_threads(caller_scratch, items, self.num_threads, f)
}
pub(crate) fn steal_for_each_mut_with_threads<T, F>(
&self,
caller_scratch: &mut CoderScratch,
items: &mut [T],
max_threads: usize,
f: F,
) where
T: Send,
F: Fn(usize, &mut T, &mut CoderScratch) + Sync,
{
self.steal_for_each_mut_with_threads_shared(caller_scratch, items, max_threads, &f)
}
fn steal_for_each_mut_with_threads_shared<T>(
&self,
caller_scratch: &mut CoderScratch,
items: &mut [T],
max_threads: usize,
f: &(dyn Fn(usize, &mut T, &mut CoderScratch) + Sync),
) where
T: Send,
{
let len = items.len();
let base = SyncPtr(items.as_mut_ptr());
let visit = move |i: usize, scratch: &mut CoderScratch| {
let item = unsafe { &mut *base.add(i) };
f(i, item, scratch);
};
self.steal_indices(caller_scratch, len, max_threads, &visit);
}
fn steal_indices(
&self,
caller_scratch: &mut CoderScratch,
len: usize,
max_threads: usize,
f: &(dyn Fn(usize, &mut CoderScratch) + Sync),
) {
let lanes = self.num_threads.min(max_threads.max(1)).min(len);
if lanes <= 1 {
for i in 0..len {
f(i, caller_scratch);
}
return;
}
let cursor = AtomicUsize::new(0);
let lane = |scratch: &mut CoderScratch| loop {
let i = cursor.fetch_add(1, Ordering::Relaxed);
if i >= len {
break;
}
f(i, scratch);
};
self.run_lanes(caller_scratch, lanes, &lane);
}
fn run_lanes(
&self,
caller_scratch: &mut CoderScratch,
lanes: usize,
lane: &(dyn Fn(&mut CoderScratch) + Sync),
) {
let completion = Arc::new(Completion {
remaining: AtomicUsize::new(lanes),
panic: Mutex::new(None),
});
for _ in 1..lanes {
let run: Box<dyn FnOnce(&mut CoderScratch) + Send + '_> = Box::new(lane);
let run = unsafe {
std::mem::transmute::<
Box<dyn FnOnce(&mut CoderScratch) + Send + '_>,
Box<dyn FnOnce(&mut CoderScratch) + Send + 'static>,
>(run)
};
self.shared.push(Task {
run,
completion: Arc::clone(&completion),
});
}
let result = catch_unwind(AssertUnwindSafe(|| lane(caller_scratch)));
completion.finish(result.err(), &self.shared);
while completion.remaining.load(Ordering::Acquire) != 0 {
if !self.shared.run_one(caller_scratch) {
self.shared.wait_for_activity(&completion.remaining);
}
}
if let Some(payload) = completion.panic.lock().unwrap().take() {
resume_unwind(payload);
}
}
}
struct SyncPtr<T>(*mut T);
impl<T> Clone for SyncPtr<T> {
fn clone(&self) -> Self {
*self
}
}
impl<T> Copy for SyncPtr<T> {}
unsafe impl<T> Sync for SyncPtr<T> {}
unsafe impl<T> Send for SyncPtr<T> {}
impl<T> SyncPtr<T> {
#[inline(always)]
fn add(self, i: usize) -> *mut T {
unsafe { self.0.add(i) }
}
}
impl Drop for ThreadPool {
fn drop(&mut self) {
{
let _queue = self.shared.queue.lock().unwrap();
self.shared.shutdown.store(true, Ordering::Release);
self.shared.activity.notify_all();
}
for worker in self.workers.drain(..) {
worker.join().unwrap();
}
}
}
fn worker_loop(shared: &Shared, scratch: &mut CoderScratch) {
loop {
let task = {
let mut queue = shared.queue.lock().unwrap();
while queue.is_empty() && !shared.shutdown.load(Ordering::Acquire) {
queue = shared.activity.wait(queue).unwrap();
}
if queue.is_empty() {
return;
}
queue.pop_front().unwrap()
};
task.execute(shared, scratch);
}
}
#[cfg(test)]
mod tests {
use super::{CoderScratch, ThreadPool};
#[test]
fn scratch_free_jobs_reuse_workers_and_respect_thread_count() {
use std::collections::HashSet;
use std::sync::Barrier;
for threads in [0, 1, 4] {
let pool = ThreadPool::new_lossless(threads);
let barrier = Barrier::new(pool.num_threads());
let workers = || {
pool.map(pool.num_threads(), |_| {
let id = std::thread::current().id();
barrier.wait();
id
})
.into_iter()
.collect::<HashSet<_>>()
};
let first = workers();
assert_eq!(first.len(), threads.max(1));
assert_eq!(workers(), first);
}
}
#[test]
fn nested_scratch_free_jobs_can_borrow_worker_scratch() {
for threads in [1, 4] {
let pool = ThreadPool::new_lossless(threads);
let mut scratch = CoderScratch::lossless();
let results = pool.steal_map(&mut scratch, 32, |outer, scratch| {
scratch.alpha_tokens.clear();
scratch
.alpha_tokens
.push(crate::entropy::Token::new(0, outer as u32));
let tokens = &scratch.alpha_tokens;
let (value, values) = pool.join(
|| tokens[0].value,
|| pool.map(17, |inner| tokens[0].value as usize * 100 + inner),
);
assert_eq!(value, outer as u32);
assert_eq!(tokens[0].value, outer as u32);
values
});
for (outer, values) in results.into_iter().enumerate() {
assert_eq!(values, (0..17).map(|i| outer * 100 + i).collect::<Vec<_>>());
}
}
}
#[test]
fn thread_creation_is_confined_to_the_pool() {
fn check_dir(dir: &std::path::Path) {
for entry in std::fs::read_dir(dir).unwrap() {
let path = entry.unwrap().path();
if path.is_dir() {
check_dir(&path);
} else if path.extension().is_some_and(|ext| ext == "rs")
&& path.file_name().unwrap() != "thread_pool.rs"
{
let source = std::fs::read_to_string(&path).unwrap();
let compact: String = source.chars().filter(|c| !c.is_whitespace()).collect();
for forbidden in [
"thread::spawn",
"thread::scope",
"thread::Builder",
"usestd::thread::{",
"usestd::thread::*",
] {
assert!(
!compact.contains(forbidden),
"{} creates threads outside ThreadPool ({forbidden})",
path.display()
);
}
}
}
}
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
for dir in [
"src",
"app/src",
"stats/src",
"meanstats/src",
"tuner/src",
"tools",
] {
let dir = root.join(dir);
if dir.is_dir() {
check_dir(&dir);
}
}
}
#[test]
fn persistent_pool_preserves_index_order() {
let pool = ThreadPool::new(4);
let mut scratch = Box::<CoderScratch>::default();
assert_eq!(
pool.steal_map(&mut scratch, 257, |i, _scratch| i.wrapping_mul(17)),
(0usize..257)
.map(|i| i.wrapping_mul(17))
.collect::<Vec<_>>()
);
}
#[test]
fn nested_maps_share_the_pool() {
let pool = ThreadPool::new(4);
let mut scratch = Box::<CoderScratch>::default();
let result = pool.steal_map(&mut scratch, 16, |outer, scratch| {
pool.steal_map_with_threads(scratch, 9, 2, |inner, _scratch| outer * 100 + inner)
});
for (outer, row) in result.iter().enumerate() {
assert_eq!(
row,
&(0..9).map(|inner| outer * 100 + inner).collect::<Vec<_>>()
);
}
}
#[test]
fn pool_can_be_reused_for_consecutive_maps() {
let pool = ThreadPool::new(3);
let mut scratch = Box::<CoderScratch>::default();
for offset in 0..8 {
assert_eq!(
pool.steal_map(&mut scratch, 31, |i, _scratch| offset + i),
(0..31).map(|i| offset + i).collect::<Vec<_>>()
);
}
}
#[test]
fn mutable_map_visits_each_item_once_without_locks() {
let pool = ThreadPool::new(4);
let mut scratch = Box::<CoderScratch>::default();
let mut values = vec![0usize; 257];
pool.steal_for_each_mut(&mut scratch, &mut values, |i, value, _scratch| {
*value = i.wrapping_mul(17);
});
assert_eq!(
values,
(0usize..257)
.map(|i| i.wrapping_mul(17))
.collect::<Vec<_>>()
);
}
#[test]
fn worker_panic_is_propagated_without_losing_the_pool() {
let pool = ThreadPool::new(4);
let mut scratch = Box::<CoderScratch>::default();
let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
pool.steal_map(&mut scratch, 64, |i, _scratch| {
assert_ne!(i, 17, "worker failure");
i
});
}));
assert!(panic.is_err());
assert_eq!(
pool.steal_map(&mut scratch, 4, |i, _scratch| i),
vec![0, 1, 2, 3]
);
}
#[test]
fn mutable_worker_panic_releases_borrows_before_reusing_the_pool() {
let pool = ThreadPool::new(4);
let mut scratch = Box::<CoderScratch>::default();
let mut values = vec![0usize; 257];
let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
pool.steal_for_each_mut(&mut scratch, &mut values, |i, value, _scratch| {
assert_ne!(i, 17, "mutable worker failure");
*value = i;
});
}));
assert!(panic.is_err());
pool.steal_for_each_mut(&mut scratch, &mut values, |i, value, _scratch| {
*value = i * 3;
});
assert_eq!(values, (0..257).map(|i| i * 3).collect::<Vec<_>>());
}
}