faultscope-collection 0.2.11

Native logical error-rate collection and scheduling for FaultScope
Documentation
//! Shared worker-pool lifecycle for collection schedulers.

use std::collections::VecDeque;
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::sync::mpsc;
use std::sync::{Arc, Condvar, Mutex};
use std::thread::{self, JoinHandle};

use faultscope_core::{NpError, NpResult};

struct WorkQueueState<W> {
    pending: VecDeque<W>,
    closed: bool,
}

struct WorkQueue<W> {
    state: Mutex<WorkQueueState<W>>,
    ready: Condvar,
}

impl<W> WorkQueue<W> {
    fn new() -> Self {
        Self {
            state: Mutex::new(WorkQueueState {
                pending: VecDeque::new(),
                closed: false,
            }),
            ready: Condvar::new(),
        }
    }

    fn send(&self, work: W) -> Result<(), mpsc::SendError<W>> {
        let mut state = self.state.lock().unwrap();
        if state.closed {
            return Err(mpsc::SendError(work));
        }
        state.pending.push_back(work);
        self.ready.notify_one();
        Ok(())
    }

    fn recv(&self) -> Option<W> {
        let mut state = self.state.lock().unwrap();
        loop {
            if let Some(work) = state.pending.pop_front() {
                return Some(work);
            }
            if state.closed {
                return None;
            }
            state = self.ready.wait(state).unwrap();
        }
    }

    fn close(&self) {
        let mut state = self.state.lock().unwrap();
        state.closed = true;
        self.ready.notify_all();
    }
}

pub(crate) struct WorkSender<W> {
    queue: Arc<WorkQueue<W>>,
}

impl<W> WorkSender<W> {
    pub(crate) fn send(&self, work: W) -> Result<(), mpsc::SendError<W>> {
        self.queue.send(work)
    }
}

impl<W> Drop for WorkSender<W> {
    fn drop(&mut self) {
        self.queue.close();
    }
}

pub(crate) struct WorkerExecutor<W, R> {
    work_tx: Option<WorkSender<W>>,
    result_rx: mpsc::Receiver<NpResult<R>>,
    handles: Vec<JoinHandle<()>>,
}

impl<W, R> WorkerExecutor<W, R>
where
    W: Send + 'static,
    R: Send + 'static,
{
    pub(crate) fn new<S, StateFactory, Execute, PanicContext, ErrorContext>(
        worker_count: usize,
        state_factory: StateFactory,
        execute: Execute,
        panic_context: PanicContext,
        error_context: ErrorContext,
    ) -> Self
    where
        S: Send + 'static,
        StateFactory: Fn() -> S + Send + Sync + 'static,
        Execute: Fn(&W, &mut S, &mpsc::Sender<NpResult<R>>) -> NpResult<R> + Send + Sync + 'static,
        PanicContext: Fn(&W) -> String + Send + Sync + 'static,
        ErrorContext: Fn(&W) -> String + Send + Sync + 'static,
    {
        let work_queue = Arc::new(WorkQueue::new());
        let work_tx = WorkSender {
            queue: work_queue.clone(),
        };
        let (result_tx, result_rx) = mpsc::channel::<NpResult<R>>();
        let state_factory = Arc::new(state_factory);
        let execute = Arc::new(execute);
        let panic_context = Arc::new(panic_context);
        let error_context = Arc::new(error_context);
        let mut handles = Vec::with_capacity(worker_count);

        for _ in 0..worker_count {
            let work_queue = work_queue.clone();
            let result_tx = result_tx.clone();
            let state_factory = state_factory.clone();
            let execute = execute.clone();
            let panic_context = panic_context.clone();
            let error_context = error_context.clone();
            handles.push(thread::spawn(move || {
                let mut state = state_factory();
                loop {
                    let Some(work) = work_queue.recv() else {
                        break;
                    };
                    let result = execute_with_context(
                        || panic_context(&work),
                        || error_context(&work),
                        || execute(&work, &mut state, &result_tx),
                    );
                    if result_tx.send(result).is_err() {
                        break;
                    }
                }
            }));
        }
        drop(result_tx);

        Self {
            work_tx: Some(work_tx),
            result_rx,
            handles,
        }
    }

    pub(crate) fn sender(&self) -> &WorkSender<W> {
        self.work_tx
            .as_ref()
            .expect("worker executor sender is available until finish")
    }

    pub(crate) fn drain<CompletesWork, OnSuccess, OnDiscard>(
        &self,
        in_flight: &mut usize,
        first_error: &mut Option<NpError>,
        result_channel_error: &str,
        completes_work: CompletesWork,
        mut on_success: OnSuccess,
        mut on_discard: OnDiscard,
    ) where
        CompletesWork: Fn(&R) -> bool,
        OnSuccess: FnMut(R, &WorkSender<W>, &mut usize) -> NpResult<()>,
        OnDiscard: FnMut(R),
    {
        while *in_flight > 0 {
            let result = match self.result_rx.recv() {
                Ok(result) => result,
                Err(_) => {
                    if first_error.is_none() {
                        *first_error = Some(NpError::new(result_channel_error));
                    }
                    break;
                }
            };
            let completes_work = match &result {
                Ok(result) => completes_work(result),
                Err(_) => true,
            };
            if completes_work {
                *in_flight -= 1;
            }

            match result {
                Ok(result) if first_error.is_none() => {
                    if let Err(err) = on_success(result, self.sender(), in_flight) {
                        *first_error = Some(err);
                    }
                }
                Ok(result) => on_discard(result),
                Err(err) if first_error.is_none() => *first_error = Some(err),
                Err(_) => {}
            }
        }
    }

    pub(crate) fn finish(
        mut self,
        mut first_error: Option<NpError>,
        join_error: &str,
    ) -> NpResult<()> {
        drop(self.work_tx.take());
        for handle in self.handles {
            if handle.join().is_err() && first_error.is_none() {
                first_error = Some(NpError::new(join_error));
            }
        }
        match first_error {
            Some(err) => Err(err),
            None => Ok(()),
        }
    }
}

pub(crate) fn execute_with_context<T, PanicContext, ErrorContext>(
    panic_context: PanicContext,
    error_context: ErrorContext,
    execute: impl FnOnce() -> NpResult<T>,
) -> NpResult<T>
where
    PanicContext: FnOnce() -> String,
    ErrorContext: FnOnce() -> String,
{
    match catch_unwind(AssertUnwindSafe(execute)) {
        Err(payload) => Err(NpError::new(format!(
            "{}: {}",
            panic_context(),
            panic_payload_message(payload.as_ref())
        ))),
        Ok(Err(err)) => Err(NpError::new(format!(
            "{}: {}",
            error_context(),
            err.message()
        ))),
        Ok(Ok(value)) => Ok(value),
    }
}

fn panic_payload_message(payload: &(dyn std::any::Any + Send)) -> &str {
    if let Some(message) = payload.downcast_ref::<&str>() {
        message
    } else if let Some(message) = payload.downcast_ref::<String>() {
        message.as_str()
    } else {
        "non-string panic payload"
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::sync::atomic::{AtomicUsize, Ordering};

    #[test]
    fn callback_can_schedule_followup_work() {
        let executor = WorkerExecutor::new(
            1,
            || (),
            |work, _, _| Ok(*work),
            |work| format!("panic on {work}"),
            |work| format!("failure on {work}"),
        );
        executor.sender().send(0usize).unwrap();
        let mut in_flight = 1usize;
        let mut first_error = None;
        let mut completed = Vec::new();

        executor.drain(
            &mut in_flight,
            &mut first_error,
            "result channel closed",
            |_| true,
            |result, work_tx, in_flight| {
                completed.push(result);
                if result < 2 {
                    work_tx.send(result + 1).unwrap();
                    *in_flight += 1;
                }
                Ok(())
            },
            |_| {},
        );

        assert_eq!(completed, vec![0, 1, 2]);
        assert_eq!(in_flight, 0);
        executor.finish(first_error, "join failed").unwrap();
    }

    #[test]
    fn panic_is_contextualized_and_joined() {
        let executor = WorkerExecutor::new(
            1,
            || (),
            |_: &(), _, _| -> NpResult<()> { panic!("decoder exploded") },
            |_| "worker panic context".to_string(),
            |_| "worker error context".to_string(),
        );
        executor.sender().send(()).unwrap();
        let mut in_flight = 1usize;
        let mut first_error = None;

        executor.drain(
            &mut in_flight,
            &mut first_error,
            "result channel closed",
            |_| true,
            |_, _, _| Ok(()),
            |_| {},
        );

        let err = executor
            .finish(first_error, "join failed")
            .expect_err("panicking work must fail");
        assert!(err.message().contains("worker panic context"));
        assert!(err.message().contains("decoder exploded"));
    }

    #[test]
    fn successful_work_does_not_build_error_context() {
        let panic_context_calls = Arc::new(AtomicUsize::new(0));
        let error_context_calls = Arc::new(AtomicUsize::new(0));
        let panic_calls = panic_context_calls.clone();
        let error_calls = error_context_calls.clone();
        let executor = WorkerExecutor::new(
            1,
            || (),
            |work, _, _| Ok(*work),
            move |_| {
                panic_calls.fetch_add(1, Ordering::SeqCst);
                "panic".to_string()
            },
            move |_| {
                error_calls.fetch_add(1, Ordering::SeqCst);
                "error".to_string()
            },
        );
        executor.sender().send(7usize).unwrap();
        let mut in_flight = 1usize;
        let mut first_error = None;
        executor.drain(
            &mut in_flight,
            &mut first_error,
            "result channel closed",
            |_| true,
            |result, _, _| {
                assert_eq!(result, 7);
                Ok(())
            },
            |_| {},
        );
        executor.finish(first_error, "join failed").unwrap();

        assert_eq!(panic_context_calls.load(Ordering::SeqCst), 0);
        assert_eq!(error_context_calls.load(Ordering::SeqCst), 0);
    }
}