use std::any::Any;
use std::collections::VecDeque;
use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Condvar, Mutex};
use std::thread::JoinHandle;
use crate::coder_scratch::CoderScratch;
struct Completion {
remaining: AtomicUsize,
panic: Mutex<Option<Box<dyn Any + Send>>>,
}
impl Completion {
fn finish(&self, panic: Option<Box<dyn Any + Send>>, shared: &Shared) {
if let Some(payload) = panic {
let mut first_panic = self.panic.lock().unwrap();
if first_panic.is_none() {
*first_panic = Some(payload);
}
}
let _queue = shared.queue.lock().unwrap();
if self.remaining.fetch_sub(1, Ordering::Release) == 1 {
shared.activity.notify_all();
}
}
}
struct Task {
run: Box<dyn FnOnce(&mut CoderScratch) + Send + 'static>,
completion: Arc<Completion>,
}
impl Task {
fn execute(self, shared: &Shared, scratch: &mut CoderScratch) {
let Self { run, completion } = self;
let result = catch_unwind(AssertUnwindSafe(|| run(scratch)));
completion.finish(result.err(), shared);
}
}
struct Shared {
queue: Mutex<VecDeque<Task>>,
activity: Condvar,
shutdown: AtomicBool,
}
impl Shared {
fn push(&self, task: Task) {
self.queue.lock().unwrap().push_back(task);
self.activity.notify_one();
}
fn run_one(&self, scratch: &mut CoderScratch) -> bool {
let task = self.queue.lock().unwrap().pop_front();
if let Some(task) = task {
task.execute(self, scratch);
true
} else {
false
}
}
fn wait_for_activity(&self, remaining: &AtomicUsize) {
let queue = self.queue.lock().unwrap();
if queue.is_empty()
&& remaining.load(Ordering::Acquire) != 0
&& !self.shutdown.load(Ordering::Acquire)
{
let _queue = self.activity.wait(queue).unwrap();
}
}
}
pub(crate) struct ThreadPool {
shared: Arc<Shared>,
workers: Vec<JoinHandle<()>>,
num_threads: usize,
}
impl ThreadPool {
pub(crate) fn new(num_threads: usize) -> Self {
Self::new_with_scratch(num_threads, CoderScratch::default)
}
pub(crate) fn new_lossless(num_threads: usize) -> Self {
Self::new_with_scratch(num_threads, CoderScratch::lossless)
}
fn new_with_scratch(num_threads: usize, make_scratch: fn() -> CoderScratch) -> Self {
let num_threads = num_threads.max(1);
let shared = Arc::new(Shared {
queue: Mutex::new(VecDeque::new()),
activity: Condvar::new(),
shutdown: AtomicBool::new(false),
});
let workers = (1..num_threads)
.map(|i| {
let shared = Arc::clone(&shared);
std::thread::Builder::new()
.name(format!("jixel-worker-{i}"))
.spawn(move || {
let mut scratch = Box::new(make_scratch());
worker_loop(&shared, &mut scratch);
})
.expect("failed to start encoder worker")
})
.collect();
Self {
shared,
workers,
num_threads,
}
}
#[inline]
pub(crate) fn num_threads(&self) -> usize {
self.num_threads
}
pub(crate) fn steal_map<T, F>(
&self,
caller_scratch: &mut CoderScratch,
len: usize,
f: F,
) -> Vec<T>
where
T: Send,
F: Fn(usize, &mut CoderScratch) -> T + Sync,
{
self.steal_map_with_threads(caller_scratch, len, self.num_threads, f)
}
pub(crate) fn steal_map_with_threads<T, F>(
&self,
caller_scratch: &mut CoderScratch,
len: usize,
max_threads: usize,
f: F,
) -> Vec<T>
where
T: Send,
F: Fn(usize, &mut CoderScratch) -> T + Sync,
{
self.steal_map_with_threads_shared(caller_scratch, len, max_threads, &f)
}
fn steal_map_with_threads_shared<T>(
&self,
caller_scratch: &mut CoderScratch,
len: usize,
max_threads: usize,
f: &(dyn Fn(usize, &mut CoderScratch) -> T + Sync),
) -> Vec<T>
where
T: Send,
{
let mut slots: Vec<Option<T>> = Vec::with_capacity(len);
slots.resize_with(len, || None);
let base = SyncPtr(slots.as_mut_ptr());
let write = move |i: usize, scratch: &mut CoderScratch| {
let value = f(i, scratch);
unsafe { *base.add(i) = Some(value) };
};
self.steal_indices(caller_scratch, len, max_threads, &write);
slots.into_iter().map(Option::unwrap).collect()
}
pub(crate) fn steal_for_each_mut<T, F>(
&self,
caller_scratch: &mut CoderScratch,
items: &mut [T],
f: F,
) where
T: Send,
F: Fn(usize, &mut T, &mut CoderScratch) + Sync,
{
self.steal_for_each_mut_with_threads(caller_scratch, items, self.num_threads, f)
}
pub(crate) fn steal_for_each_mut_with_threads<T, F>(
&self,
caller_scratch: &mut CoderScratch,
items: &mut [T],
max_threads: usize,
f: F,
) where
T: Send,
F: Fn(usize, &mut T, &mut CoderScratch) + Sync,
{
self.steal_for_each_mut_with_threads_shared(caller_scratch, items, max_threads, &f)
}
fn steal_for_each_mut_with_threads_shared<T>(
&self,
caller_scratch: &mut CoderScratch,
items: &mut [T],
max_threads: usize,
f: &(dyn Fn(usize, &mut T, &mut CoderScratch) + Sync),
) where
T: Send,
{
let len = items.len();
let base = SyncPtr(items.as_mut_ptr());
let visit = move |i: usize, scratch: &mut CoderScratch| {
let item = unsafe { &mut *base.add(i) };
f(i, item, scratch);
};
self.steal_indices(caller_scratch, len, max_threads, &visit);
}
fn steal_indices(
&self,
caller_scratch: &mut CoderScratch,
len: usize,
max_threads: usize,
f: &(dyn Fn(usize, &mut CoderScratch) + Sync),
) {
let lanes = self.num_threads.min(max_threads.max(1)).min(len);
if lanes <= 1 {
for i in 0..len {
f(i, caller_scratch);
}
return;
}
let cursor = AtomicUsize::new(0);
let lane = |scratch: &mut CoderScratch| loop {
let i = cursor.fetch_add(1, Ordering::Relaxed);
if i >= len {
break;
}
f(i, scratch);
};
self.run_lanes(caller_scratch, lanes, &lane);
}
fn run_lanes(
&self,
caller_scratch: &mut CoderScratch,
lanes: usize,
lane: &(dyn Fn(&mut CoderScratch) + Sync),
) {
let completion = Arc::new(Completion {
remaining: AtomicUsize::new(lanes),
panic: Mutex::new(None),
});
for _ in 1..lanes {
let run: Box<dyn FnOnce(&mut CoderScratch) + Send + '_> = Box::new(lane);
let run = unsafe {
std::mem::transmute::<
Box<dyn FnOnce(&mut CoderScratch) + Send + '_>,
Box<dyn FnOnce(&mut CoderScratch) + Send + 'static>,
>(run)
};
self.shared.push(Task {
run,
completion: Arc::clone(&completion),
});
}
let result = catch_unwind(AssertUnwindSafe(|| lane(caller_scratch)));
completion.finish(result.err(), &self.shared);
while completion.remaining.load(Ordering::Acquire) != 0 {
if !self.shared.run_one(caller_scratch) {
self.shared.wait_for_activity(&completion.remaining);
}
}
if let Some(payload) = completion.panic.lock().unwrap().take() {
resume_unwind(payload);
}
}
}
struct SyncPtr<T>(*mut T);
impl<T> Clone for SyncPtr<T> {
fn clone(&self) -> Self {
*self
}
}
impl<T> Copy for SyncPtr<T> {}
unsafe impl<T> Sync for SyncPtr<T> {}
unsafe impl<T> Send for SyncPtr<T> {}
impl<T> SyncPtr<T> {
#[inline(always)]
fn add(self, i: usize) -> *mut T {
unsafe { self.0.add(i) }
}
}
impl Drop for ThreadPool {
fn drop(&mut self) {
{
let _queue = self.shared.queue.lock().unwrap();
self.shared.shutdown.store(true, Ordering::Release);
self.shared.activity.notify_all();
}
for worker in self.workers.drain(..) {
worker.join().unwrap();
}
}
}
fn worker_loop(shared: &Shared, scratch: &mut CoderScratch) {
loop {
let task = {
let mut queue = shared.queue.lock().unwrap();
while queue.is_empty() && !shared.shutdown.load(Ordering::Acquire) {
queue = shared.activity.wait(queue).unwrap();
}
if queue.is_empty() {
return;
}
queue.pop_front().unwrap()
};
task.execute(shared, scratch);
}
}
#[cfg(test)]
mod tests {
use super::{CoderScratch, ThreadPool};
#[test]
fn persistent_pool_preserves_index_order() {
let pool = ThreadPool::new(4);
let mut scratch = Box::<CoderScratch>::default();
assert_eq!(
pool.steal_map(&mut scratch, 257, |i, _scratch| i.wrapping_mul(17)),
(0usize..257)
.map(|i| i.wrapping_mul(17))
.collect::<Vec<_>>()
);
}
#[test]
fn nested_maps_share_the_pool() {
let pool = ThreadPool::new(4);
let mut scratch = Box::<CoderScratch>::default();
let result = pool.steal_map(&mut scratch, 16, |outer, scratch| {
pool.steal_map_with_threads(scratch, 9, 2, |inner, _scratch| outer * 100 + inner)
});
for (outer, row) in result.iter().enumerate() {
assert_eq!(
row,
&(0..9).map(|inner| outer * 100 + inner).collect::<Vec<_>>()
);
}
}
#[test]
fn pool_can_be_reused_for_consecutive_maps() {
let pool = ThreadPool::new(3);
let mut scratch = Box::<CoderScratch>::default();
for offset in 0..8 {
assert_eq!(
pool.steal_map(&mut scratch, 31, |i, _scratch| offset + i),
(0..31).map(|i| offset + i).collect::<Vec<_>>()
);
}
}
#[test]
fn mutable_map_visits_each_item_once_without_locks() {
let pool = ThreadPool::new(4);
let mut scratch = Box::<CoderScratch>::default();
let mut values = vec![0usize; 257];
pool.steal_for_each_mut(&mut scratch, &mut values, |i, value, _scratch| {
*value = i.wrapping_mul(17);
});
assert_eq!(
values,
(0usize..257)
.map(|i| i.wrapping_mul(17))
.collect::<Vec<_>>()
);
}
#[test]
fn worker_panic_is_propagated_without_losing_the_pool() {
let pool = ThreadPool::new(4);
let mut scratch = Box::<CoderScratch>::default();
let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
pool.steal_map(&mut scratch, 64, |i, _scratch| {
assert_ne!(i, 17, "worker failure");
i
});
}));
assert!(panic.is_err());
assert_eq!(
pool.steal_map(&mut scratch, 4, |i, _scratch| i),
vec![0, 1, 2, 3]
);
}
#[test]
fn mutable_worker_panic_releases_borrows_before_reusing_the_pool() {
let pool = ThreadPool::new(4);
let mut scratch = Box::<CoderScratch>::default();
let mut values = vec![0usize; 257];
let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
pool.steal_for_each_mut(&mut scratch, &mut values, |i, value, _scratch| {
assert_ne!(i, 17, "mutable worker failure");
*value = i;
});
}));
assert!(panic.is_err());
pool.steal_for_each_mut(&mut scratch, &mut values, |i, value, _scratch| {
*value = i * 3;
});
assert_eq!(values, (0..257).map(|i| i * 3).collect::<Vec<_>>());
}
}