vthread 0.1.0

Bounded carrier-affine virtual threads for Rust
Documentation
//! Multi-producer and multi-carrier coverage around the remote receive window.

use crate::{
    CarrierId, Error, Runtime,
    control::Shared,
    signal::lock,
    support_test::{run_isolated, wait_without_intervention},
};
use std::{
    io::Write,
    sync::{
        Arc,
        atomic::{AtomicUsize, Ordering},
        mpsc,
    },
    thread,
    time::{Duration, Instant},
};

const PRODUCERS: usize = 4;
const TASKS_PER_PRODUCER: usize = 129;
const TASKS: usize = PRODUCERS * TASKS_PER_PRODUCER;
const QUEUE_CAPACITY: usize = 65;
const WATCHDOG: Duration = Duration::from_secs(5);
const CHILD: &str = "VTHREAD_REFILL_MATRIX_CHILD";

#[derive(Default)]
struct Counts {
    accepted: AtomicUsize,
    returned: AtomicUsize,
    retries: AtomicUsize,
    executed: [AtomicUsize; 2],
}

struct Stop(Arc<Shared>);

impl Drop for Stop {
    fn drop(&mut self) {
        self.0.request_stop();
    }
}

fn isolate() -> bool {
    if std::env::var(CHILD).as_deref() == Ok("1") {
        return false;
    }
    let name = "carrier::carrier_refill_matrix_test::small_queues_progress_with_multiple_producers_and_carriers";
    let output = run_isolated(name, (CHILD, "1"), Duration::from_secs(25));
    let stdout = String::from_utf8_lossy(&output.stdout);
    let stderr = String::from_utf8_lossy(&output.stderr);
    assert!(
        !output.timed_out && output.status.success() && stdout.contains("1 passed"),
        "isolated refill matrix failed: status={:?} timed_out={}\nstdout:\n{stdout}\nstderr:\n{stderr}",
        output.status,
        output.timed_out,
    );
    true
}

fn record(shared: &Shared, counts: &Counts, phase: &str) {
    let pending = shared
        .inboxes
        .iter()
        .map(|inbox| inbox.pending())
        .collect::<Vec<_>>();
    let epochs = shared
        .inboxes
        .iter()
        .map(|inbox| inbox.signal.version())
        .collect::<Vec<_>>();
    let carriers = shared
        .inboxes
        .iter()
        .map(|inbox| inbox.signal.test_progress.snapshot())
        .collect::<Vec<_>>();
    let mut output = std::io::stdout().lock();
    writeln!(
        output,
        "refill-matrix phase={phase} accepted={} returned={} retries={} \
         pending={pending:?} epochs={epochs:?} carriers={carriers:?}",
        counts.accepted.load(Ordering::Acquire),
        counts.returned.load(Ordering::Acquire),
        counts.retries.load(Ordering::Acquire),
    )
    .unwrap();
    output.flush().unwrap();
}

#[test]
fn small_queues_progress_with_multiple_producers_and_carriers() {
    if isolate() {
        return;
    }
    let config = Runtime::builder()
        .carriers(2)
        .max_vthreads(TASKS)
        .carrier_queue_capacity(QUEUE_CAPACITY)
        .build()
        .unwrap()
        .config();
    let shared = Arc::new(Shared::new(config));
    for inbox in &shared.inboxes {
        inbox.signal.test_progress.enable();
    }
    let scope = shared.begin_scope().unwrap();
    let counts = Arc::new(Counts::default());
    let outcome = thread::scope(|threads| {
        let stop = Stop(Arc::clone(&shared));
        let (hooked, hooked_rx) = mpsc::channel();
        let releases = shared
            .inboxes
            .iter()
            .enumerate()
            .map(|(index, inbox)| {
                let hooked = hooked.clone();
                let (release, release_rx) = mpsc::channel();
                *lock(&inbox.before_notify_hook) = Some(Box::new(move || {
                    let _ = hooked.send(index);
                    let _ = release_rx.recv();
                }));
                release
            })
            .collect::<Vec<_>>();
        drop(hooked);
        let (done, done_rx) = mpsc::channel();
        let producers = (0..PRODUCERS)
            .map(|producer| {
                let shared = Arc::clone(&shared);
                let counts = Arc::clone(&counts);
                let done = done.clone();
                threads.spawn(move || {
                    let result = fill(&shared, scope, producer, &counts);
                    let _ = done.send(result);
                })
            })
            .collect::<Vec<_>>();
        drop(done);
        let hook_deadline = Instant::now() + WATCHDOG;
        let mut hook_order = Vec::new();
        for _ in 0..2 {
            if let Ok(index) =
                hooked_rx.recv_timeout(hook_deadline.saturating_duration_since(Instant::now()))
            {
                hook_order.push(index);
            }
        }
        hook_order.sort_unstable();
        if hook_order == [0, 1] {
            let fill_deadline = Instant::now() + WATCHDOG;
            while (shared
                .inboxes
                .iter()
                .any(|inbox| inbox.pending() != QUEUE_CAPACITY)
                || counts.retries.load(Ordering::Acquire) == 0)
                && Instant::now() < fill_deadline
            {
                thread::yield_now();
            }
        }
        let queues_full = shared
            .inboxes
            .iter()
            .all(|inbox| inbox.pending() == QUEUE_CAPACITY);
        let retried = counts.retries.load(Ordering::Acquire) != 0;
        record(&shared, &counts, "before-notification");
        for release in releases {
            let _ = release.send(());
        }
        // Start the consumers only after bounded admission has filled both
        // inboxes; later publishers now wake already-running carriers.
        let carriers = (0..2)
            .map(|index| {
                let carrier = Arc::clone(&shared);
                threads.spawn(move || super::run(carrier, CarrierId(index)))
            })
            .collect::<Vec<_>>();
        assert_eq!(hook_order, [0, 1], "both notifier hooks must pause");
        assert!(
            queues_full,
            "both queues must fill while notification is paused"
        );
        assert!(retried, "full queues must reject at least one admission");
        let deadline = Instant::now() + WATCHDOG;
        let admission: Result<(), String> = (0..PRODUCERS).try_for_each(|_| {
            done_rx
                .recv_timeout(deadline.saturating_duration_since(Instant::now()))
                .map_err(|error| format!("{error:?}"))?
                .map_err(|error| format!("{error:?}"))
        });
        let drained = admission
            .is_ok()
            .then(|| shared.wait_until(scope, None, Some(Instant::now() + WATCHDOG)));
        let passed = admission.is_ok() && matches!(drained, Some(Ok(true)));
        record(&shared, &counts, "primary");
        if !passed {
            wait_without_intervention(Duration::from_secs(1));
            record(&shared, &counts, "no-intervention");
        }
        let report = shared.scope_report(scope);
        let snapshot = shared.snapshot();
        let queued = shared
            .inboxes
            .iter()
            .map(|inbox| inbox.pending())
            .sum::<usize>();
        writeln!(
            std::io::stdout().lock(),
            "refill-matrix-rich admission={admission:?} drained={drained:?} \
             report={report:?} snapshot={snapshot:?}"
        )
        .unwrap();
        drop(stop);
        for producer in producers {
            producer.join().unwrap();
        }
        for carrier in carriers {
            carrier.join().unwrap();
        }
        (admission, drained, report, snapshot, queued)
    });
    shared.finish_scope(scope);
    assert!(
        outcome.0.is_ok(),
        "matrix admission failed: {:?}",
        outcome.0
    );
    assert!(matches!(outcome.1, Some(Ok(true))), "matrix did not drain");
    assert_eq!(counts.accepted.load(Ordering::Acquire), TASKS);
    assert_eq!(counts.returned.load(Ordering::Acquire), TASKS);
    assert!(
        counts
            .executed
            .iter()
            .all(|count| count.load(Ordering::Acquire) != 0)
    );
    assert_eq!(
        (outcome.2.completed, outcome.3.active, outcome.4),
        (TASKS as u64, 0, 0)
    );
    assert!(outcome.2.failures.is_empty(), "{:?}", outcome.2);
    assert!(
        shared
            .inboxes
            .iter()
            .all(|inbox| inbox.reclaimed.load(Ordering::Acquire))
    );
}

fn fill(shared: &Shared, scope: u64, producer: usize, counts: &Arc<Counts>) -> crate::Result<()> {
    for index in 0..TASKS_PER_PRODUCER {
        loop {
            let body = Arc::clone(counts);
            match shared.submit(scope, format!("producer-{producer}-{index}"), move || {
                let carrier = crate::worker_context::current_carrier().unwrap();
                body.executed[carrier.0].fetch_add(1, Ordering::Release);
                body.returned.fetch_add(1, Ordering::Release);
            }) {
                Ok(_) => {
                    counts.accepted.fetch_add(1, Ordering::Release);
                    break;
                }
                Err(Error::Capacity {
                    resource: crate::error::CapacityResource::CarrierQueue,
                    ..
                }) => {
                    counts.retries.fetch_add(1, Ordering::Relaxed);
                    thread::yield_now();
                }
                Err(error) => return Err(error),
            }
        }
    }
    Ok(())
}