use super::block::{CODEC_STORED, CODEC_ZSTD, CODEC_ZSTD_DICT};
use anyhow::Result;
#[cfg(not(target_arch = "wasm32"))]
use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
use std::sync::Arc;
#[cfg(not(target_arch = "wasm32"))]
use std::sync::Mutex;
pub struct Job {
pub block_id: u32,
pub raw: Arc<Vec<u8>>,
}
pub struct Done {
pub block_id: u32,
pub codec: u8,
pub raw: Arc<Vec<u8>>,
pub payload: Vec<u8>,
}
#[cfg(not(target_arch = "wasm32"))]
pub struct Pool {
tx: Option<SyncSender<Job>>,
rx: Mutex<Receiver<Done>>,
workers: Vec<std::thread::JoinHandle<()>>,
pub outstanding: usize,
}
#[cfg(not(target_arch = "wasm32"))]
impl Pool {
pub fn new(threads: usize, level: i32, dict: Option<Arc<Vec<u8>>>) -> Pool {
let threads = threads.max(1);
let (jtx, jrx) = sync_channel::<Job>(threads * 2);
let (dtx, drx) = sync_channel::<Done>(threads * 4);
let jrx = Arc::new(std::sync::Mutex::new(jrx));
let mut workers = Vec::with_capacity(threads);
for _ in 0..threads {
let jrx = jrx.clone();
let dtx = dtx.clone();
let dict = dict.clone();
workers.push(std::thread::spawn(move || loop {
let job = {
let g = jrx.lock().unwrap();
match g.recv() {
Ok(j) => j,
Err(_) => return,
}
};
let (codec, payload) =
match super::codec::encode(&job.raw, dict.as_deref().map(|v| &v[..]), level) {
Ok((c, p)) => (c, p.into_owned()),
Err(_) => (CODEC_STORED, job.raw.as_ref().clone()),
};
debug_assert!(matches!(codec, CODEC_STORED | CODEC_ZSTD | CODEC_ZSTD_DICT));
if dtx.send(Done { block_id: job.block_id, codec, raw: job.raw, payload }).is_err()
{
return;
}
}));
}
Pool { tx: Some(jtx), rx: Mutex::new(drx), workers, outstanding: 0 }
}
pub fn submit(&mut self, block_id: u32, raw: Arc<Vec<u8>>) -> Result<()> {
self.tx
.as_ref()
.expect("pool is shut down")
.send(Job { block_id, raw })
.map_err(|_| anyhow::anyhow!("compression pool died"))?;
self.outstanding += 1;
Ok(())
}
pub fn try_take(&mut self) -> Vec<Done> {
let mut out = Vec::new();
while let Ok(d) = self.rx.lock().unwrap().try_recv() {
self.outstanding -= 1;
out.push(d);
}
out
}
pub fn take_all(&mut self) -> Result<Vec<Done>> {
let mut out = Vec::new();
while self.outstanding > 0 {
let d = self
.rx
.lock()
.unwrap()
.recv()
.map_err(|_| anyhow::anyhow!("compression pool died with work outstanding"))?;
self.outstanding -= 1;
out.push(d);
}
Ok(out)
}
}
#[cfg(not(target_arch = "wasm32"))]
impl Drop for Pool {
fn drop(&mut self) {
self.tx.take();
for w in self.workers.drain(..) {
let _ = w.join();
}
}
}
#[cfg(target_arch = "wasm32")]
pub struct Pool {
done: std::collections::VecDeque<Done>,
level: i32,
dict: Option<Arc<Vec<u8>>>,
pub outstanding: usize,
}
#[cfg(target_arch = "wasm32")]
impl Pool {
pub fn new(_threads: usize, level: i32, dict: Option<Arc<Vec<u8>>>) -> Pool {
Pool { done: std::collections::VecDeque::new(), level, dict, outstanding: 0 }
}
pub fn submit(&mut self, block_id: u32, raw: Arc<Vec<u8>>) -> Result<()> {
let (codec, payload) =
match super::codec::encode(&raw, self.dict.as_deref().map(|v| &v[..]), self.level) {
Ok((c, p)) => (c, p.into_owned()),
Err(_) => (CODEC_STORED, raw.as_ref().clone()),
};
debug_assert!(matches!(codec, CODEC_STORED | CODEC_ZSTD | CODEC_ZSTD_DICT));
self.done.push_back(Done { block_id, codec, raw, payload });
self.outstanding += 1;
Ok(())
}
pub fn try_take(&mut self) -> Vec<Done> {
let out: Vec<Done> = self.done.drain(..).collect();
self.outstanding -= out.len();
out
}
pub fn take_all(&mut self) -> Result<Vec<Done>> {
Ok(self.try_take())
}
}