use std::cell::{Cell, RefCell};
use std::ffi::CStr;
use vj_abi::eval::{EvalOutcome, EvalResult, RescheduleReq};
use vj_abi::{ProcessFn, SimHandle};
use super::reschedule_req_proposal::{
KIND_BY_REQUEST, KIND_DELAY, KIND_DONE, KIND_EDGE, KIND_EDGE_LIST, KIND_FORK_JOIN,
KIND_REPEAT_EDGE, KIND_WAIT_FORK, RescheduleReqLayoutProposal,
};
use super::{
CompiledProcess, EventTerm, EventWaitSet, JoinKind, ProcessId, ProcessRequest, ProcessState,
ResumeArg, SchedulerOps, SpawnSpec, Suspend,
};
use crate::dpi_task::{CallPoll, ExportRequest, TaskCall};
use crate::process::{ActivationRecord, ArenaRange, PredicateFn, WaitSpecStorage};
use crate::state::{EdgeKind, FourStateValue, InstanceHandle, StateCellId};
const NEXT_RESUME_PC_OFFSET: usize = 24;
const FLAGS_OFFSET: usize = 28;
const FLAG_RECHECKS_ON_WAKE: u8 = 1;
const PROPOSAL_SIZE: usize = 24;
#[derive(Clone, Copy, Debug)]
pub enum ResolvedSuspend {
Delay(u64),
DynamicDelay,
Edge {
cell: StateCellId,
edge: EdgeKind,
},
Done,
RepeatEdge {
count: u64,
cell: StateCellId,
edge: EdgeKind,
},
EdgeWithGuard {
cell: StateCellId,
edge: EdgeKind,
guard_fn: extern "C" fn(SimHandle) -> bool,
},
ForkJoin {
join_kind: JoinKind,
base: u32,
count: u32,
},
WaitFork,
ByRequest,
EventList {
base: u32,
count: u32,
},
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct WaitTableEntry {
pub slot: u32,
pub edge: Option<EdgeKind>,
pub select: Option<(u32, u32)>,
}
impl WaitTableEntry {
pub fn whole(slot: u32, edge: Option<EdgeKind>) -> Self {
Self {
slot,
edge,
select: None,
}
}
}
pub fn build_eval_result(resolved: ResolvedSuspend, next_resume_pc: u32) -> EvalResult {
let (outcome, proposal) = match resolved {
ResolvedSuspend::Delay(ticks) => (
EvalOutcome::SuspendedOnDelay,
RescheduleReqLayoutProposal::from_delay(ticks),
),
ResolvedSuspend::DynamicDelay => (
EvalOutcome::SuspendedOnDelay,
RescheduleReqLayoutProposal::from_delay(DYNAMIC_DELAY_MARK),
),
ResolvedSuspend::Edge { cell, edge } => (
EvalOutcome::SuspendedOnEvent,
RescheduleReqLayoutProposal::from_single_edge(edge, cell),
),
ResolvedSuspend::Done => (EvalOutcome::Completed, RescheduleReqLayoutProposal::done()),
ResolvedSuspend::RepeatEdge { count, cell, edge } => (
EvalOutcome::SuspendedOnEvent,
RescheduleReqLayoutProposal::from_repeat_edge(count, edge, cell),
),
ResolvedSuspend::EdgeWithGuard {
cell,
edge,
guard_fn,
} => (
EvalOutcome::SuspendedOnEvent,
RescheduleReqLayoutProposal::from_single_edge_with_guard(edge, cell, guard_fn),
),
ResolvedSuspend::ForkJoin {
join_kind,
base,
count,
} => (
EvalOutcome::SuspendedOnEvent,
RescheduleReqLayoutProposal::from_fork_join(join_kind, ArenaRange { base, count }),
),
ResolvedSuspend::WaitFork => (
EvalOutcome::SuspendedOnEvent,
RescheduleReqLayoutProposal::wait_fork(),
),
ResolvedSuspend::ByRequest => (
EvalOutcome::SuspendedOnEvent,
RescheduleReqLayoutProposal::by_request(),
),
ResolvedSuspend::EventList { base, count } => (
EvalOutcome::SuspendedOnEvent,
RescheduleReqLayoutProposal::from_edge_list(ArenaRange { base, count }),
),
};
let mut opaque = [0u8; vj_abi::eval::RESCHEDULE_REQ_SIZE];
let proposal_bytes: [u8; PROPOSAL_SIZE] = proposal.to_bytes();
opaque[..PROPOSAL_SIZE].copy_from_slice(&proposal_bytes);
opaque[NEXT_RESUME_PC_OFFSET..NEXT_RESUME_PC_OFFSET + 4]
.copy_from_slice(&next_resume_pc.to_le_bytes());
EvalResult {
outcome,
reschedule: RescheduleReq { opaque },
}
}
pub fn mark_rechecks_on_wake(mut er: EvalResult) -> EvalResult {
er.reschedule.opaque[FLAGS_OFFSET] |= FLAG_RECHECKS_ON_WAKE;
er
}
fn rechecks_on_wake(er: &EvalResult) -> bool {
er.reschedule.opaque[FLAGS_OFFSET] & FLAG_RECHECKS_ON_WAKE != 0
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum DecodeRescheduleError {
UnresolvedEdgeList { base: u32, count: u32 },
UnresolvedForkJoin {
join_kind: u8,
base: u32,
count: u32,
},
InvalidKind(u8),
}
pub fn decode_reschedule(r: EvalResult) -> Result<Suspend, DecodeRescheduleError> {
let mut proposal_bytes = [0u8; PROPOSAL_SIZE];
proposal_bytes.copy_from_slice(&r.reschedule.opaque[..PROPOSAL_SIZE]);
let proposal: RescheduleReqLayoutProposal = unsafe { core::mem::transmute(proposal_bytes) };
match proposal.kind {
KIND_DELAY => {
let ticks = u64::from(proposal.a) | (u64::from(proposal.b) << 32);
if ticks == DYNAMIC_DELAY_MARK {
return Ok(Suspend::Delay(DYNAMIC_DELAY.with(Cell::get)));
}
Ok(Suspend::Delay(ticks))
}
KIND_EDGE => Ok(Suspend::WaitEvents(EventWaitSet {
terms: vec![EventTerm::Edge {
cell: StateCellId(proposal.edge_signal),
edge: decode_edge_kind_u8(proposal.edge_kind),
iff: proposal.guard_fn().map(PredicateFn),
}],
rechecks_on_wake: rechecks_on_wake(&r),
})),
KIND_DONE => Ok(Suspend::Done),
KIND_WAIT_FORK => Ok(Suspend::WaitFork),
KIND_BY_REQUEST => Ok(Suspend::ByRequest),
KIND_REPEAT_EDGE => Ok(Suspend::WaitEvents(EventWaitSet {
terms: vec![EventTerm::Repeat {
count: u64::from(proposal.a) | (u64::from(proposal.b) << 32),
inner: Box::new(EventTerm::Edge {
cell: StateCellId(proposal.edge_signal),
edge: decode_edge_kind_u8(proposal.edge_kind),
iff: None,
}),
}],
..Default::default()
})),
KIND_EDGE_LIST => Err(DecodeRescheduleError::UnresolvedEdgeList {
base: proposal.a,
count: proposal.b,
}),
KIND_FORK_JOIN => Err(DecodeRescheduleError::UnresolvedForkJoin {
join_kind: proposal.fork_join_kind,
base: proposal.a,
count: proposal.b,
}),
other => Err(DecodeRescheduleError::InvalidKind(other)),
}
}
pub fn decode_reschedule_with_fork_table(
r: EvalResult,
fork_table: &[SpawnSpec],
) -> Result<Suspend, DecodeRescheduleError> {
decode_reschedule_with_tables(r, fork_table, &[])
}
pub fn decode_reschedule_with_tables(
r: EvalResult,
fork_table: &[SpawnSpec],
wait_table: &[WaitTableEntry],
) -> Result<Suspend, DecodeRescheduleError> {
match decode_reschedule(r) {
Err(DecodeRescheduleError::UnresolvedEdgeList { base, count }) => {
let start = base as usize;
let end = start.saturating_add(count as usize);
let entries = wait_table
.get(start..end)
.ok_or(DecodeRescheduleError::UnresolvedEdgeList { base, count })?;
Ok(Suspend::WaitEvents(EventWaitSet {
terms: entries
.iter()
.map(|e| match (e.select, e.edge) {
(Some((lsb, width)), edge) => EventTerm::Select {
cell: StateCellId(u64::from(e.slot)),
edge: edge.unwrap_or(EdgeKind::None),
lsb,
width,
},
(None, Some(edge)) => EventTerm::Edge {
cell: StateCellId(u64::from(e.slot)),
edge,
iff: None,
},
(None, None) => {
EventTerm::ImplicitAny(vec![StateCellId(u64::from(e.slot))])
}
})
.collect(),
rechecks_on_wake: rechecks_on_wake(&r),
}))
}
other => decode_fork_join(other, fork_table),
}
}
fn decode_fork_join(
decoded: Result<Suspend, DecodeRescheduleError>,
fork_table: &[SpawnSpec],
) -> Result<Suspend, DecodeRescheduleError> {
match decoded {
Err(DecodeRescheduleError::UnresolvedForkJoin {
join_kind,
base,
count,
}) => {
let start = base as usize;
let end = start.saturating_add(count as usize);
let spawns =
fork_table
.get(start..end)
.ok_or(DecodeRescheduleError::UnresolvedForkJoin {
join_kind,
base,
count,
})?;
Ok(Suspend::ForkJoin(
decode_join_kind_u8(join_kind),
spawns.to_vec(),
))
}
other => other,
}
}
fn decode_join_kind_u8(b: u8) -> JoinKind {
if b == JoinKind::All as u8 {
JoinKind::All
} else if b == JoinKind::Any as u8 {
JoinKind::Any
} else {
JoinKind::None
}
}
fn decode_edge_kind_u8(b: u8) -> EdgeKind {
if b == EdgeKind::Posedge as u8 {
EdgeKind::Posedge
} else if b == EdgeKind::Negedge as u8 {
EdgeKind::Negedge
} else if b == EdgeKind::BothEdges as u8 {
EdgeKind::BothEdges
} else {
EdgeKind::None
}
}
pub const RETURN_COPY: u32 = 1 << 31;
type ReturnBox = std::rc::Rc<RefCell<Option<Vec<(u32, u64)>>>>;
pub struct CompiledKernelProcess {
pub kernel: ProcessFn,
pub activation: Box<ActivationRecord>,
pub resume_pc: u32,
frame: Box<[u8]>,
sim_handle: SimHandle,
fork_table: std::sync::Arc<[SpawnSpec]>,
fork_kernels: std::sync::Arc<[Option<ForkKernel>]>,
map_slots_to_instance: bool,
wait_table: std::sync::Arc<[WaitTableEntry]>,
disable_instance: Option<u32>,
disable_scopes: std::sync::Arc<[(u32, String, u32)]>,
joined: bool,
finished: bool,
task_fibers: bool,
start_inputs: Option<Vec<i64>>,
call: Option<TaskCall>,
fiber_result: std::rc::Rc<Cell<Option<EvalResult>>>,
request: Option<ExportRequest>,
image_back: Option<(usize, usize)>,
returns: Vec<(u32, u32)>,
return_box: Option<ReturnBox>,
pending_returns: RefCell<Vec<ReturnBox>>,
}
#[derive(Clone, Debug)]
pub struct ForkKernel {
pub kernel: ProcessFn,
pub frame_word_count: u32,
pub captures: std::sync::Arc<[(u32, u32)]>,
pub disable_scopes: std::sync::Arc<[(u32, String, u32)]>,
}
unsafe impl Send for CompiledKernelProcess {}
impl CompiledKernelProcess {
pub fn new(kernel: ProcessFn, frame_word_count: u32, sim_handle: SimHandle) -> Self {
let byte_len = frame_word_count as usize * 8;
Self {
kernel,
activation: Box::new(ActivationRecord {
pc: 0,
frame: ArenaRange {
base: 0,
count: frame_word_count,
},
pending_wait: WaitSpecStorage::None,
fork_state: None,
}),
resume_pc: 0,
frame: vec![0u8; byte_len].into_boxed_slice(),
sim_handle,
fork_table: std::sync::Arc::from(Vec::new()),
fork_kernels: std::sync::Arc::from(Vec::new()),
map_slots_to_instance: false,
wait_table: std::sync::Arc::from(Vec::new()),
disable_instance: None,
disable_scopes: std::sync::Arc::from(Vec::new()),
joined: false,
finished: false,
task_fibers: false,
start_inputs: None,
call: None,
fiber_result: std::rc::Rc::new(Cell::new(None)),
request: None,
image_back: None,
returns: Vec::new(),
return_box: None,
pending_returns: RefCell::new(Vec::new()),
}
}
pub fn with_task_fibers(mut self) -> Self {
self.task_fibers = true;
self
}
pub fn with_disable_scopes(
mut self,
instance: u32,
scopes: std::sync::Arc<[(u32, String, u32)]>,
) -> Self {
self.disable_instance = Some(instance);
self.disable_scopes = scopes;
self
}
pub fn with_start_inputs(mut self, inputs: Vec<i64>) -> Self {
self.start_inputs = Some(inputs);
self
}
pub fn with_wait_table(mut self, wait_table: std::sync::Arc<[WaitTableEntry]>) -> Self {
self.wait_table = wait_table;
self
}
pub fn with_instance_slot_mapping(mut self) -> Self {
self.map_slots_to_instance = true;
self
}
pub fn with_fork_table(mut self, fork_table: std::sync::Arc<[SpawnSpec]>) -> Self {
self.fork_table = fork_table;
self
}
pub fn with_fork_kernels(mut self, fork_kernels: std::sync::Arc<[Option<ForkKernel>]>) -> Self {
self.fork_kernels = fork_kernels;
self
}
pub fn frame_bytes(&self) -> &[u8] {
&self.frame
}
}
impl CompiledProcess for CompiledKernelProcess {
fn mint_fork_child(&self, spec: &SpawnSpec) -> Option<Box<dyn CompiledProcess>> {
let fk = self.fork_kernels.get(spec.kernel as usize)?.clone()?;
let mut child = CompiledKernelProcess::new(fk.kernel, fk.frame_word_count, self.sim_handle)
.with_fork_table(self.fork_table.clone())
.with_fork_kernels(self.fork_kernels.clone())
.with_wait_table(self.wait_table.clone());
child.map_slots_to_instance = self.map_slots_to_instance;
child.task_fibers = self.task_fibers;
if let Some(req) = self.request
&& let Some(row) = self.fork_table.get(req.export as usize)
&& row.kernel == spec.kernel
&& row.entry_pc == spec.entry_pc
{
if !req.handle.is_null() {
child.sim_handle = SimHandle(req.handle);
}
let n = child.frame.len();
unsafe {
std::ptr::copy_nonoverlapping(req.args, child.frame.as_mut_ptr(), n);
}
child.image_back = Some((req.args as usize, n));
}
child.disable_instance = self.disable_instance;
child.disable_scopes = fk.disable_scopes.clone();
for &(p, c) in fk.captures.iter() {
if p & RETURN_COPY != 0 {
child.returns.push((c, p & !RETURN_COPY));
continue;
}
let (p, c) = (p as usize * 8, c as usize * 8);
if p + 8 <= self.frame.len() && c + 8 <= child.frame.len() {
child.frame[c..c + 8].copy_from_slice(&self.frame[p..p + 8]);
}
}
if !child.returns.is_empty() {
let bx: ReturnBox = std::rc::Rc::new(RefCell::new(None));
child.return_box = Some(bx.clone());
self.pending_returns.borrow_mut().push(bx);
}
Some(Box::new(child))
}
fn disable_instance(&self) -> Option<u32> {
self.disable_instance
}
fn disable_exit(&self, label: &str) -> Option<u32> {
if self.finished {
return None;
}
if let Some(call) = &self.call {
return (self.request.is_some() && call.import_name() == label)
.then_some(self.resume_pc);
}
self.disable_scopes
.iter()
.find(|(pc, l, _)| *pc == self.resume_pc && l == label)
.map(|(_, _, exit)| *exit)
}
fn owns_disable_scope(&self, label: &str) -> bool {
self.disable_scopes.iter().any(|(_, l, _)| l == label)
}
fn redirect_resume(&mut self, pc: u32) -> bool {
if let Some(call) = self.call.as_mut() {
call.disable();
self.request = None;
return std::mem::take(&mut self.joined);
}
self.resume_pc = pc;
self.activation.pc = pc;
std::mem::take(&mut self.joined)
}
fn resume(&mut self, _state: SimHandle, arg: ResumeArg, ops: &dyn SchedulerOps) -> Suspend {
if matches!(
arg,
ResumeArg::EventFired { .. } | ResumeArg::WaitOrderSatisfied | ResumeArg::Disabled
) && let Some(proc) = current_process()
{
crate::sched::deferred::flush_deferred(proc);
}
if !matches!(arg, ResumeArg::Start | ResumeArg::TimeElapsed)
&& let Some(proc) = current_process()
{
crate::sched::proc_assert::flush(proc);
}
if matches!(
arg,
ResumeArg::EventFired { .. } | ResumeArg::WaitOrderSatisfied
) && let Some(proc) = current_process()
{
crate::sched::violation::flush_violation_reports(proc);
}
if let Some(inputs) = self.start_inputs.take() {
crate::sched::proc_assert::set_action_inputs(inputs);
}
ABORTED.with(|c| c.set(false));
let mask = current_process().map_or(0, take_fired);
FIRED_MASK.with(|c| c.set(mask));
FIRED_TERM.with(|c| {
c.set(match arg {
ResumeArg::EventFired { term_index } => term_index as i64,
_ => -1,
})
});
if !self.pending_returns.get_mut().is_empty() {
let frame = &mut self.frame;
self.pending_returns.get_mut().retain(|bx| {
let Some(words) = bx.borrow_mut().take() else {
return true;
};
for (p, v) in words {
let at = p as usize * 8;
if at + 8 <= frame.len() {
frame[at..at + 8].copy_from_slice(&v.to_le_bytes());
}
}
false
});
}
let local_ptr = if self.frame.is_empty() {
std::ptr::null_mut()
} else {
self.frame.as_mut_ptr()
};
let handle = self.sim_handle;
let result: EvalResult = if self.task_fibers {
if self.call.is_none() {
let (kernel, pc, out) = (self.kernel, self.resume_pc, self.fiber_result.clone());
let local = local_ptr as usize;
self.call = TaskCall::new(move || {
out.set(Some(kernel(handle, local as *mut _, pc)));
0
});
assert!(
self.call.is_some(),
"no stack could be mapped for a process fiber"
);
}
let call = self.call.as_mut().expect("created above");
match with_scheduler_ops(ops, || call.poll()) {
CallPoll::Finished(_) => {
self.call = None;
self.request = None;
self.fiber_result
.take()
.expect("a finished process fiber left no result")
}
CallPoll::Request(req) => {
let row = self.fork_table.get(req.export as usize).unwrap_or_else(|| {
panic!(
"export task {} has no fork-table row (internal)",
req.export
)
});
let spec = row.clone();
self.request = Some(req);
self.joined = true;
return Suspend::ForkJoin(JoinKind::All, vec![spec]);
}
}
} else {
with_scheduler_ops(ops, || (self.kernel)(handle, local_ptr, self.resume_pc))
};
let mut pc_bytes = [0u8; 4];
pc_bytes.copy_from_slice(
&result.reschedule.opaque[NEXT_RESUME_PC_OFFSET..NEXT_RESUME_PC_OFFSET + 4],
);
self.resume_pc = u32::from_le_bytes(pc_bytes);
self.activation.pc = self.resume_pc;
let suspend = decode_reschedule_with_tables(result, &self.fork_table, &self.wait_table)
.unwrap_or_else(|e| {
unreachable!(
"CompiledKernelProcess::resume: decode_reschedule_with_fork_table returned \
{e:?}, but build_eval_result (the only producer of RescheduleReq bytes in \
this tree) never constructs an EdgeList/invalid kind, and a real \
KIND_FORK_JOIN's (base,count) always indexes this same instance's own \
fork_table (an internal invariant of how vj-codegen pairs a kernel \
with its program's fork table -- see CompiledKernelProcess::with_fork_table)"
)
});
self.joined = matches!(
suspend,
Suspend::ForkJoin(JoinKind::All | JoinKind::Any, _)
| Suspend::WaitFork
| Suspend::WaitOrder(_)
);
self.finished = matches!(suspend, Suspend::Done);
if self.finished
&& let Some(bx) = self.return_box.take()
{
let words = self
.returns
.iter()
.map(|&(c, p)| {
let at = c as usize * 8;
let mut w = [0u8; 8];
w.copy_from_slice(&self.frame[at..at + 8]);
(p, u64::from_le_bytes(w))
})
.collect();
*bx.borrow_mut() = Some(words);
}
if self.finished
&& let Some((image, n)) = self.image_back.take()
{
unsafe {
std::ptr::copy_nonoverlapping(self.frame.as_ptr(), image as *mut u8, n);
}
}
if !self.map_slots_to_instance {
return suspend;
}
match suspend {
Suspend::WaitEvents(mut set) => {
for term in &mut set.terms {
remap_term_to_instance(term, self.sim_handle);
}
Suspend::WaitEvents(set)
}
other => other,
}
}
}
fn remap_term_to_instance(term: &mut EventTerm, handle: SimHandle) {
let map = |c: &mut StateCellId| {
if *c == OBJECT_STATE_CELL {
OBJECT_STATE_WATCHED.store(true, std::sync::atomic::Ordering::Relaxed);
} else if *c != SHARED_VARS_CELL {
*c = crate::state::cell_for_slot(handle, vj_abi::SlotId(c.0 as u32));
}
};
match term {
EventTerm::Edge { cell, .. } | EventTerm::Select { cell, .. } => map(cell),
EventTerm::ImplicitAny(cells) => cells.iter_mut().for_each(map),
EventTerm::NamedEventTrigger(cell) => map(cell),
EventTerm::Repeat { inner, .. } => remap_term_to_instance(inner, handle),
}
}
const DYNAMIC_DELAY_MARK: u64 = u64::MAX;
thread_local! {
static DYNAMIC_DELAY: Cell<u64> = const { Cell::new(0) };
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_set_delay"))]
pub extern "C" fn real_sv_rt_set_delay(ticks: u64) {
DYNAMIC_DELAY.with(|c| c.set(ticks));
}
thread_local! {
static FIRED_TERM: Cell<i64> = const { Cell::new(-1) };
static FIRED_MASKS: std::cell::RefCell<Vec<u64>> = const { std::cell::RefCell::new(Vec::new()) };
static FIRED_MASK: Cell<u64> = const { Cell::new(0) };
static ABORTED: Cell<bool> = const { Cell::new(false) };
}
thread_local! {
static CELLS: std::cell::RefCell<Vec<Box<[u64]>>> = const { std::cell::RefCell::new(Vec::new()) };
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_cell_alloc"))]
pub extern "C" fn real_sv_rt_cell_alloc(bytes: u64) -> *mut u8 {
let words = (bytes as usize).div_ceil(8).max(1);
let mut cell: Box<[u64]> = vec![0u64; words].into_boxed_slice();
let ptr = cell.as_mut_ptr().cast::<u8>();
CELLS.with(|c| c.borrow_mut().push(cell));
ptr
}
pub const SHARED_VARS_SLOT: u32 = u32::MAX;
pub const SHARED_VARS_CELL: crate::state::StateCellId =
crate::state::StateCellId(SHARED_VARS_SLOT as u64);
pub const OBJECT_STATE_SLOT: u32 = u32::MAX - 1;
pub const OBJECT_STATE_CELL: crate::state::StateCellId =
crate::state::StateCellId(OBJECT_STATE_SLOT as u64);
static OBJECT_STATE_WATCHED: std::sync::atomic::AtomicBool =
std::sync::atomic::AtomicBool::new(false);
pub fn watch_object_state() {
OBJECT_STATE_WATCHED.store(true, std::sync::atomic::Ordering::Relaxed);
}
pub(crate) fn object_state_watched() -> bool {
OBJECT_STATE_WATCHED.load(std::sync::atomic::Ordering::Relaxed)
}
pub fn notify_object_state_change() {
if OBJECT_STATE_WATCHED.load(std::sync::atomic::Ordering::Relaxed) {
with_current_ops(|ops| ops.notify_blocking_row_change(OBJECT_STATE_CELL));
}
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_cell_store"))]
pub unsafe extern "C" fn real_sv_rt_cell_store(cell: *mut u8, value: *const u8, bytes: u64) {
let (to, from) = unsafe {
(
std::slice::from_raw_parts_mut(cell, bytes as usize),
std::slice::from_raw_parts(value, bytes as usize),
)
};
if to != from {
to.copy_from_slice(from);
with_current_ops(|ops| ops.notify_blocking_row_change(SHARED_VARS_CELL));
}
}
thread_local! {
static EDGE_SNAPSHOTS: std::cell::RefCell<std::collections::HashMap<(u64, u64), Vec<u8>>> =
std::cell::RefCell::new(std::collections::HashMap::new());
}
type ElemWaits = std::collections::HashMap<ProcessId, Vec<(u64, StateCellId, u64)>>;
thread_local! {
static ELEM_WAITS: std::cell::RefCell<ElemWaits> =
std::cell::RefCell::new(std::collections::HashMap::new());
}
pub const ELEM_SITE_BIT: u64 = 1 << 62;
pub fn event_elem_waiters(cell: StateCellId, row: u64) -> Vec<ProcessId> {
ELEM_WAITS.with(|m| {
let mut v: Vec<ProcessId> = m
.borrow()
.iter()
.filter(|(_, rows)| rows.iter().any(|&(_, c, r)| c == cell && r == row))
.map(|(p, _)| *p)
.collect();
v.sort_unstable();
v
})
}
fn snapshot_key(frame: u64, site: u64) -> (u64, u64) {
match frame {
0 => (
current_process().map_or(0, |p| (1 << 63) | u64::from(p)),
site,
),
_ => (frame, site),
}
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_edge_snapshot"))]
pub unsafe extern "C" fn real_sv_rt_edge_snapshot(
frame: u64,
site: u64,
value: *const u8,
bytes: u64,
) {
let v = unsafe { std::slice::from_raw_parts(value, bytes as usize) }.to_vec();
if site & ELEM_SITE_BIT != 0
&& bytes == 40
&& let Some(proc) = current_process()
{
let word = |i: usize| u64::from_le_bytes(v[i * 8..i * 8 + 8].try_into().unwrap());
let wait_of = |site: u64| site >> 16;
ELEM_WAITS.with(|m| {
let mut m = m.borrow_mut();
let rows = m.entry(proc).or_default();
rows.retain(|&(s, ..)| wait_of(s) == wait_of(site) && s != site);
if word(1) != 0 {
let handle = SimHandle(word(4) as usize as *mut std::ffi::c_void);
let cell = crate::state::cell_for_slot(handle, vj_abi::SlotId(word(3) as u32));
rows.push((site, cell, word(0)));
}
});
}
EDGE_SNAPSHOTS.with(|m| m.borrow_mut().insert(snapshot_key(frame, site), v));
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_edge_check"))]
pub unsafe extern "C" fn real_sv_rt_edge_check(
frame: u64,
site: u64,
value: *const u8,
bytes: u64,
width: u32,
edge: u32,
) -> i32 {
let new = unsafe { std::slice::from_raw_parts(value, bytes as usize) }.to_vec();
let bit = |buf: &[u8], i: usize| buf.get(i / 8).is_some_and(|b| b >> (i % 8) & 1 == 1);
let used_bits = 2 * width as usize;
let changed = |old: &[u8]| (0..used_bits).any(|i| bit(old, i) != bit(&new, i));
let key = snapshot_key(frame, site);
let fired = EDGE_SNAPSHOTS.with(|m| {
let mut m = m.borrow_mut();
let fired = match m.get(&key) {
None => false,
Some(old) => {
let state = |buf: &[u8]| {
let (v, u) = (bit(buf, 0), bit(buf, width as usize));
(!u && !v, !u && v, u)
};
let (o0, o1, ox) = state(old);
let (n0, n1, nx) = state(&new);
let pos = (o0 && (n1 || nx)) || (ox && n1);
let neg = (o1 && (n0 || nx)) || (ox && n0);
match edge {
1 => pos,
2 => neg,
3 => pos || neg,
_ => changed(old),
}
}
};
m.insert(key, new.clone());
fired
});
i32::from(fired)
}
pub unsafe extern "C" fn real_sv_rt_edge_load(frame: u64, site: u64, out: *mut u8, bytes: u64) {
let dst = unsafe { std::slice::from_raw_parts_mut(out, bytes as usize) };
dst.fill(0);
EDGE_SNAPSHOTS.with(|m| {
if let Some(v) = m.borrow().get(&snapshot_key(frame, site)) {
let n = v.len().min(dst.len());
dst[..n].copy_from_slice(&v[..n]);
}
});
}
pub extern "C" fn real_sv_rt_set_fired_term(term: i64) {
FIRED_TERM.with(|c| c.set(term));
}
pub fn reset_cells_for_new_run() {
CELLS.with(|c| c.borrow_mut().clear());
crate::state::reset_driver_counts_for_new_run();
EDGE_SNAPSHOTS.with(|m| m.borrow_mut().clear());
ELEM_WAITS.with(|m| m.borrow_mut().clear());
super::semaphore::reset_for_new_run();
super::class_event::reset_for_new_run();
super::mailbox::reset_for_new_run();
crate::sched::proc_assert::reset();
crate::sched::assert_control::reset();
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_aborted"))]
pub extern "C" fn real_sv_rt_aborted() -> i32 {
ABORTED.with(|c| i32::from(c.get()))
}
pub(crate) fn note_fired_term(proc: u32, term: usize) {
if term < 64 {
FIRED_MASKS.with(|m| {
let mut m = m.borrow_mut();
let i = proc as usize;
if i >= m.len() {
m.resize(i + 1, 0);
}
m[i] |= 1u64 << term;
});
}
}
fn take_fired(proc: u32) -> u64 {
FIRED_MASKS.with(|m| {
m.borrow_mut()
.get_mut(proc as usize)
.map_or(0, std::mem::take)
})
}
pub fn take_fired_mask() -> u64 {
current_process().map_or(0, take_fired)
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_fired_mask"))]
pub extern "C" fn real_sv_rt_fired_mask() -> i64 {
FIRED_MASK.with(|c| c.get()) as i64
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_fired_term"))]
pub extern "C" fn real_sv_rt_fired_term() -> i64 {
FIRED_TERM.with(|c| c.get())
}
thread_local! {
static CURRENT_OPS: RefCell<Option<*const (dyn SchedulerOps + 'static)>> =
const { RefCell::new(None) };
static CURRENT_PROCESS: RefCell<Option<ProcessId>> = const { RefCell::new(None) };
static REACTIVE_DRIVE: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
static EDGE_DRIVE: std::cell::Cell<Option<EdgeWait>> = const { std::cell::Cell::new(None) };
}
#[derive(Clone, Copy)]
pub(super) struct EdgeWait {
pub(super) cell: StateCellId,
pub(super) edge: EdgeKind,
pub(super) after: u64,
pub(super) on_event: bool,
}
pub(super) fn reactive_drive() -> bool {
REACTIVE_DRIVE.with(std::cell::Cell::get)
}
pub(super) fn edge_drive() -> Option<EdgeWait> {
EDGE_DRIVE.with(std::cell::Cell::get)
}
pub(super) const EDGE_DELAY: u64 = u64::MAX;
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_reactive_drive"))]
pub unsafe extern "C" fn real_sv_rt_reactive_drive(
handle: SimHandle,
on: u32,
clock_slot: u32,
edge: u32,
after: u64,
) {
if on != 0 {
with_current_ops(|ops| ops.note_reactive_drive());
}
REACTIVE_DRIVE.with(|c| c.set(on != 0));
let wait = (on != 0 && edge != 0).then(|| EdgeWait {
cell: crate::state::cell_for_slot(handle, vj_abi::SlotId(clock_slot)),
edge: match edge {
1 => EdgeKind::Posedge,
2 => EdgeKind::Negedge,
_ => EdgeKind::BothEdges,
},
after,
on_event: edge == 4,
});
EDGE_DRIVE.with(|c| c.set(wait));
}
pub fn with_current_process<R>(proc: ProcessId, f: impl FnOnce() -> R) -> R {
let prev = CURRENT_PROCESS.with(|c| c.replace(Some(proc)));
let result = f();
CURRENT_PROCESS.with(|c| *c.borrow_mut() = prev);
result
}
pub fn current_process() -> Option<ProcessId> {
CURRENT_PROCESS.with(|c| *c.borrow())
}
pub fn with_scheduler_ops<R>(ops: &dyn SchedulerOps, f: impl FnOnce() -> R) -> R {
let raw: *const dyn SchedulerOps = ops;
let raw_static: *const (dyn SchedulerOps + 'static) = unsafe { std::mem::transmute(raw) };
let prev = CURRENT_OPS.with(|c| c.replace(Some(raw_static)));
let result = f();
CURRENT_OPS.with(|c| *c.borrow_mut() = prev);
result
}
pub fn with_current_ops(f: impl FnOnce(&dyn SchedulerOps)) {
let raw = CURRENT_OPS.with(|c| *c.borrow());
if let Some(raw) = raw {
let ops: &dyn SchedulerOps = unsafe { &*raw };
f(ops);
}
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "nba_enqueue"))]
pub extern "C" fn real_nba_enqueue(
handle: SimHandle,
slot: vj_abi::SlotId,
value: u128,
source: u32,
) {
let ih = unsafe { InstanceHandle::from_sim_handle(handle) };
let cell = ih.cell_for_slot(slot.0 as usize);
let source = CURRENT_PROCESS.with(|c| *c.borrow()).unwrap_or(source);
with_current_ops(|ops| ops.stage_nba(cell, FourStateValue::Narrow(value), source));
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_nba_masked"))]
pub unsafe extern "C" fn real_sv_rt_nba_masked(
handle: SimHandle,
slot: u32,
row: u64,
has_row: u32,
val: *const u64,
unk: *const u64,
mask: *const u64,
nwords: u32,
width: u32,
delay: u64,
) {
let ih = unsafe { InstanceHandle::from_sim_handle(handle) };
let cell = ih.cell_for_slot(slot as usize);
let n = nwords as usize;
let (val, unk, mask) = unsafe {
(
std::slice::from_raw_parts(val, n).to_vec(),
std::slice::from_raw_parts(unk, n).to_vec(),
std::slice::from_raw_parts(mask, n).to_vec(),
)
};
let value = if width <= 64 {
FourStateValue::Narrow(u128::from(val[0]) | (u128::from(unk[0]) << 64))
} else {
FourStateValue::Wide {
aval: val.iter().zip(&unk).map(|(v, u)| v ^ u).collect(),
bval: unk,
}
};
let row = (has_row != 0).then_some(row);
let source = CURRENT_PROCESS.with(|c| *c.borrow()).unwrap_or(0);
with_current_ops(|ops| {
if delay == 0 {
ops.stage_nba_masked(cell, row, value, mask, source);
} else {
ops.stage_nba_masked_delayed(cell, row, value, mask, source, delay);
}
});
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_nba_coll"))]
pub unsafe extern "C" fn real_sv_rt_nba_coll(
handle: SimHandle,
slot: u32,
row: u64,
has_row: u32,
index: i64,
flags: u32,
elem: *const u64,
elem_words: u32,
delay: u64,
key: *const u64,
key_kind: u32,
key_width: u32,
) {
let ih = unsafe { InstanceHandle::from_sim_handle(handle) };
let cell = ih.cell_for_slot(slot as usize);
let elem = unsafe { std::slice::from_raw_parts(elem, elem_words as usize) }.to_vec();
let obj = flags & 2 != 0;
let mut shape = crate::collections::Shape {
elem_words: elem_words as usize,
assoc: false,
key: crate::collections::KeyKind::Int {
width: 0,
signed: false,
},
obj,
nested: crate::collections::Nested::NONE,
};
let mut key_val = None;
if !key.is_null() {
shape.assoc = true;
shape.key = if key_kind == 2 {
crate::collections::KeyKind::Str
} else if key_kind == 3 {
crate::collections::KeyKind::Obj
} else {
crate::collections::KeyKind::Int {
width: key_width,
signed: key_kind == 1,
}
};
let kw = if key_kind == 2 {
1
} else {
(2 * key_width as usize).div_ceil(64).max(1)
};
let bits = unsafe { std::slice::from_raw_parts(key, kw) };
match shape.key_from_bits(bits) {
Some(k) => key_val = Some(k),
None => {
shape.discard(&elem);
return;
}
}
}
let coll = super::nba::CollWrite {
shape,
index,
key: key_val,
queue: flags & 1 != 0,
};
let row = (has_row != 0).then_some(row);
let source = CURRENT_PROCESS.with(|c| *c.borrow()).unwrap_or(0);
with_current_ops(|ops| ops.stage_nba_coll(cell, row, elem, coll, source, delay));
}
pub unsafe extern "C" fn real_sv_rt_nba_field(
h: u64,
offset: u64,
words: u32,
val: *const u64,
mask: *const u64,
delay: u64,
) {
if !crate::objects::is_live(h) {
return;
}
let n = words as usize;
let (val, mask) = unsafe {
(
std::slice::from_raw_parts(val, n).to_vec(),
std::slice::from_raw_parts(mask, n).to_vec(),
)
};
let w = super::nba::ObjWrite::new(h, offset as usize, val, mask);
let source = CURRENT_PROCESS.with(|c| *c.borrow()).unwrap_or(0);
with_current_ops(|ops| ops.stage_nba_obj(w, source, delay));
}
pub fn real_sv_rt_nba_class_event(id: u64, delay: u64) {
if id == 0 {
return;
}
let w = super::nba::ObjWrite::class_event(id);
let source = CURRENT_PROCESS.with(|c| *c.borrow()).unwrap_or(0);
with_current_ops(|ops| ops.stage_nba_obj(w, source, delay));
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_nba_pulse"))]
pub unsafe extern "C" fn real_sv_rt_nba_pulse(
handle: SimHandle,
slot: u32,
row: u64,
has_row: u32,
delay: u64,
) {
let ih = unsafe { InstanceHandle::from_sim_handle(handle) };
let cell = ih.cell_for_slot(slot as usize);
let row = (has_row != 0).then_some(row);
let source = CURRENT_PROCESS.with(|c| *c.borrow()).unwrap_or(0);
with_current_ops(|ops| ops.stage_nba_pulse(cell, row, source, delay));
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_set_scalar"))]
pub extern "C" fn real_sv_set_scalar(handle: SimHandle, slot: vj_abi::SlotId, value: u128) {
let ih = unsafe { InstanceHandle::from_sim_handle(handle) };
let cell = ih.cell_for_slot(slot.0 as usize);
with_current_ops(|ops| ops.commit_blocking_write(cell, FourStateValue::Narrow(value)));
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_set_wide"))]
pub extern "C" fn real_sv_set_wide(
handle: SimHandle,
slot: vj_abi::SlotId,
value: vj_abi::WideView,
) {
let ih = unsafe { InstanceHandle::from_sim_handle(handle) };
let cell = ih.cell_for_slot(slot.0 as usize);
let n = value.word_count as usize;
let aval = unsafe { std::slice::from_raw_parts(value.aval, n) }.to_vec();
let bval = unsafe { std::slice::from_raw_parts(value.bval, n) }.to_vec();
with_current_ops(|ops| ops.commit_blocking_write(cell, FourStateValue::Wide { aval, bval }));
}
pub const HUGE_OP_PRIME: u32 = 0;
pub const HUGE_OP_MARK: u32 = 1;
pub const HUGE_OP_COMMIT: u32 = 2;
pub const HUGE_OP_NBA: u32 = 3;
#[allow(clippy::too_many_arguments)]
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_huge"))]
pub unsafe extern "C" fn real_sv_rt_huge(
op: u32,
handle: SimHandle,
slot: u32,
width: u32,
a: i64,
b: i64,
p: *const u64,
delay: u64,
) -> *mut u64 {
match op {
HUGE_OP_NBA => {
if b > 0 && !p.is_null() {
let img = unsafe { std::slice::from_raw_parts(p, (2 * b as usize).div_ceil(64)) };
if let Some(w) =
crate::state::huge_nba_window(handle, vj_abi::SlotId(slot), width, a, b, img)
{
let value = FourStateValue::Wide {
aval: w.aval,
bval: w.bval,
};
let source = CURRENT_PROCESS.with(|c| *c.borrow()).unwrap_or(0);
with_current_ops(|ops| {
ops.stage_nba_window(w.cell, w.window, value, w.mask, source, delay);
});
}
}
std::ptr::null_mut()
}
HUGE_OP_PRIME => crate::state::huge_prime(handle, vj_abi::SlotId(slot), width, a != 0),
HUGE_OP_MARK => {
crate::state::huge_mark(handle, vj_abi::SlotId(slot), a, b);
std::ptr::null_mut()
}
HUGE_OP_COMMIT => {
huge_commit_and_notify(handle, slot, width);
std::ptr::null_mut()
}
_ => std::ptr::null_mut(),
}
}
fn huge_commit_and_notify(handle: SimHandle, slot: u32, width: u32) {
use crate::state::HugeCommit;
let slot = vj_abi::SlotId(slot);
let wants = |cell| {
let answer = std::cell::Cell::new(false);
with_current_ops(|ops| answer.set(ops.wants_delta(cell)));
answer.get()
};
match crate::state::huge_commit(handle, slot, width, &wants) {
HugeCommit::Nothing | HugeCommit::Unchanged => {}
HugeCommit::Changed {
cell,
old,
new,
delta,
} => {
let edge = crate::sched::edge_between(old, new);
with_current_ops(|ops| ops.notify_blocking_wide_change(cell, edge, delta));
}
HugeCommit::Full {
cell,
value,
revalidate,
} => {
with_current_ops(|ops| ops.commit_blocking_write(cell, value));
crate::state::huge_commit_full_done(handle, slot, revalidate);
}
}
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_set_mem_word"))]
pub extern "C" fn real_sv_set_mem_word(
handle: SimHandle,
slot: vj_abi::SlotId,
row_index: u64,
value: vj_abi::WideView,
) {
let (handle, slot) = crate::state::resolve_foreign_slot(handle, slot);
let (cell, changed) =
crate::state::set_mem_word_reporting_change(handle, slot, row_index, value);
if changed {
with_current_ops(|ops| ops.notify_blocking_row_change(cell));
}
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_finish"))]
pub extern "C" fn real_sv_rt_finish(_handle: SimHandle, code: i32) {
ABORTED.with(|c| c.set(true));
with_current_ops(|ops| ops.request_finish(code));
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_disable_fork"))]
pub extern "C" fn real_sv_rt_disable_fork(_handle: SimHandle) {
if let Some(proc) = current_process() {
with_current_ops(|ops| ops.disable_fork(proc));
}
}
#[cfg_attr(
feature = "aot-exports",
unsafe(export_name = "sv_rt_disable_fork_block")
)]
pub extern "C" fn real_sv_rt_disable_fork_block(_handle: SimHandle) {
if let Some(proc) = current_process() {
with_current_ops(|ops| ops.disable_fork_block(proc));
}
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_disable_named"))]
pub unsafe extern "C" fn real_sv_rt_disable_named(
_handle: SimHandle,
label: *const std::ffi::c_char,
) {
let label = unsafe { CStr::from_ptr(label) }
.to_str()
.expect("sv_rt_disable_named: codegen must pass a valid UTF-8 C string");
if let Some(proc) = current_process() {
let mut ends_caller = false;
with_current_ops(|ops| ends_caller = ops.disable_named(proc, label));
if ends_caller {
abort_current_kernel();
}
}
}
pub fn process_class_request(req: ProcessRequest) -> bool {
let Some(requester) = current_process() else {
return false;
};
let mut ends_caller = false;
with_current_ops(|ops| ends_caller = ops.process_request(requester, req));
ends_caller
}
pub fn process_class_state(target: ProcessId) -> ProcessState {
let mut state = ProcessState::Running;
with_current_ops(|ops| state = ops.process_state(target));
state
}
pub fn process_class_track_current() -> Option<ProcessId> {
let proc = current_process()?;
with_current_ops(|ops| ops.process_track(proc));
Some(proc)
}
pub fn abort_current_kernel() {
ABORTED.with(|c| c.set(true));
}
unsafe fn force_window(
total: u32,
wide: bool,
lo: u32,
width: u32,
val: *const u64,
unk: *const u64,
) -> (FourStateValue, Vec<u64>) {
let n = width.div_ceil(64).max(1) as usize;
let (val, unk) = unsafe {
(
std::slice::from_raw_parts(val, n),
std::slice::from_raw_parts(unk, n),
)
};
let words = total.div_ceil(64).max(1) as usize;
let (mut v, mut u, mut m) = (vec![0u64; words], vec![0u64; words], vec![0u64; words]);
for i in 0..width {
let (src, dst) = (i as usize, (lo + i) as usize);
if dst >= total as usize {
break;
}
let bit = |w: &[u64]| (w[src / 64] >> (src % 64)) & 1;
v[dst / 64] |= bit(val) << (dst % 64);
u[dst / 64] |= bit(unk) << (dst % 64);
m[dst / 64] |= 1 << (dst % 64);
}
let value = if wide {
FourStateValue::Wide {
aval: v.iter().zip(&u).map(|(a, b)| a ^ b).collect(),
bval: u,
}
} else {
FourStateValue::Narrow(u128::from(v[0]) | (u128::from(u[0]) << 64))
};
(value, m)
}
fn slot_state_or_abort(
handle: SimHandle,
slot: vj_abi::SlotId,
) -> Option<(StateCellId, u32, bool, FourStateValue, u32)> {
match crate::state::slot_state(handle, slot) {
Ok(state) => Some(state),
Err(what) => {
eprintln!("vj: runtime error: {what} is not supported");
real_sv_rt_finish(handle, 1);
None
}
}
}
pub const FORCE_PROCEDURAL_ASSIGN: u32 = 2;
pub const FORCE_REFRESH: u32 = 4;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ForceEvent {
pub cell: StateCellId,
pub release: bool,
pub procedural: bool,
}
thread_local! {
static FORCE_EVENTS: RefCell<Option<Vec<ForceEvent>>> = const { RefCell::new(None) };
}
pub fn record_force_events(on: bool) {
FORCE_EVENTS.with(|e| *e.borrow_mut() = on.then(Vec::new));
}
pub fn take_force_events() -> Vec<ForceEvent> {
FORCE_EVENTS.with(|e| {
e.borrow_mut()
.as_mut()
.map(std::mem::take)
.unwrap_or_default()
})
}
fn note_force(ev: ForceEvent) {
FORCE_EVENTS.with(|e| {
if let Some(v) = e.borrow_mut().as_mut() {
v.push(ev);
}
});
}
#[allow(clippy::too_many_arguments)]
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_force"))]
pub unsafe extern "C" fn real_sv_rt_force(
handle: SimHandle,
slot: vj_abi::SlotId,
lo: u32,
width: u32,
val: *const u64,
unk: *const u64,
flags: u32,
site: u64,
) {
if let Some((row, lo_base)) = crate::state::slot_row_alias(handle, slot) {
unsafe {
force_row_impl(
handle,
slot,
row,
lo + lo_base,
width,
val,
unk,
flags & !1,
site,
);
}
return;
}
let Some((cell, total, wide, current, lo_base)) = slot_state_or_abort(handle, slot) else {
return;
};
let lo = lo + lo_base;
let (value, mask) = unsafe { force_window(total, wide, lo, width, val, unk) };
let site = (handle.0 as u64, site);
if flags & FORCE_REFRESH == 0 {
note_force(ForceEvent {
cell,
release: false,
procedural: flags & FORCE_PROCEDURAL_ASSIGN != 0,
});
}
let write = crate::state::with_force_table(handle, |t| {
let w = t.force(
cell,
value,
mask,
flags & 1 != 0,
site,
current,
flags & FORCE_PROCEDURAL_ASSIGN != 0,
);
t.set_bypass(true);
w
});
with_current_ops(|ops| ops.commit_blocking_write(cell, write));
crate::state::with_force_table(handle, |t| t.set_bypass(false));
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_release"))]
pub extern "C" fn real_sv_rt_release(
handle: SimHandle,
slot: vj_abi::SlotId,
lo: u32,
width: u32,
flags: u32,
) {
if let Some((row, lo_base)) = crate::state::slot_row_alias(handle, slot) {
release_row_impl(handle, slot, row, lo + lo_base, width, flags);
return;
}
let Some((cell, total, wide, _, lo_base)) = slot_state_or_abort(handle, slot) else {
return;
};
let lo = lo + lo_base;
note_force(ForceEvent {
cell,
release: true,
procedural: flags & FORCE_PROCEDURAL_ASSIGN != 0,
});
let ones = vec![u64::MAX; width.div_ceil(64).max(1) as usize];
let (_, mask) = unsafe { force_window(total, wide, lo, width, ones.as_ptr(), ones.as_ptr()) };
let procedural = flags & FORCE_PROCEDURAL_ASSIGN != 0;
if let Some(restore) =
crate::state::with_force_table(handle, |t| t.release(cell, &mask, procedural))
{
crate::state::with_force_table(handle, |t| t.set_bypass(true));
with_current_ops(|ops| ops.commit_blocking_write(cell, restore));
crate::state::with_force_table(handle, |t| t.set_bypass(false));
}
}
#[allow(clippy::too_many_arguments)]
unsafe fn force_row_impl(
handle: SimHandle,
slot: vj_abi::SlotId,
row: u64,
lo: u32,
width: u32,
val: *const u64,
unk: *const u64,
flags: u32,
site: u64,
) {
let (cell, at, total, current) = crate::state::row_state(handle, slot, row);
let (value, mask) = unsafe { force_window(total, true, lo, width, val, unk) };
let procedural = flags & FORCE_PROCEDURAL_ASSIGN != 0;
if flags & FORCE_REFRESH == 0 {
note_force(ForceEvent {
cell,
release: false,
procedural,
});
}
let site = (handle.0 as u64, site);
let write = crate::state::with_force_table(handle, |t| {
let w = t.force_row(
cell,
at,
value,
mask,
flags & 1 != 0,
site,
current,
procedural,
);
t.set_bypass(true);
w
});
commit_row(handle, slot, row, write);
crate::state::with_force_table(handle, |t| t.set_bypass(false));
}
fn release_row_impl(
handle: SimHandle,
slot: vj_abi::SlotId,
row: u64,
lo: u32,
width: u32,
flags: u32,
) {
let (cell, at, total, _) = crate::state::row_state(handle, slot, row);
let procedural = flags & FORCE_PROCEDURAL_ASSIGN != 0;
note_force(ForceEvent {
cell,
release: true,
procedural,
});
let ones = vec![u64::MAX; width.div_ceil(64).max(1) as usize];
let (_, mask) = unsafe { force_window(total, true, lo, width, ones.as_ptr(), ones.as_ptr()) };
if let Some(restore) =
crate::state::with_force_table(handle, |t| t.release_row(cell, at, &mask, procedural))
{
commit_row(handle, slot, row, restore);
}
}
#[allow(clippy::too_many_arguments)]
pub unsafe fn vpi_force_row(
handle: SimHandle,
slot: vj_abi::SlotId,
row: u64,
lo: u32,
width: u32,
val: *const u64,
unk: *const u64,
is_net: bool,
site: u64,
) {
unsafe {
force_row_impl(
handle,
slot,
row,
lo,
width,
val,
unk,
u32::from(is_net),
site,
);
}
}
pub fn vpi_release_row(handle: SimHandle, slot: vj_abi::SlotId, row: u64, lo: u32, width: u32) {
release_row_impl(handle, slot, row, lo, width, 0);
}
pub const FORCE_ROW_RELEASE: u32 = 8;
#[allow(clippy::too_many_arguments)]
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_force_row"))]
pub unsafe extern "C" fn real_sv_rt_force_row(
handle: SimHandle,
slot: vj_abi::SlotId,
row: u64,
lo: u32,
width: u32,
val: *const u64,
unk: *const u64,
flags: u32,
site: u64,
) {
if flags & FORCE_ROW_RELEASE != 0 {
release_row_impl(handle, slot, row, lo, width, flags);
} else {
unsafe { force_row_impl(handle, slot, row, lo, width, val, unk, flags, site) };
}
}
fn commit_row(handle: SimHandle, slot: vj_abi::SlotId, row: u64, v: FourStateValue) {
let FourStateValue::Wide { mut aval, mut bval } = v else {
return;
};
let view = vj_abi::accessor::WideView {
aval: aval.as_mut_ptr(),
bval: bval.as_mut_ptr(),
word_count: aval.len() as u32,
};
let (handle, slot) = crate::state::resolve_foreign_slot(handle, slot);
let (cell, changed) = crate::state::set_mem_word_reporting_change(handle, slot, row, view);
if changed {
with_current_ops(|ops| ops.notify_blocking_row_change(cell));
}
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_force_active"))]
pub extern "C" fn real_sv_rt_force_active(handle: SimHandle, site: u64) -> u32 {
u32::from(crate::state::with_force_table(handle, |t| {
t.site_active((handle.0 as u64, site))
}))
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_net_drivers"))]
pub unsafe extern "C" fn real_sv_rt_net_drivers(
handle: SimHandle,
slot: vj_abi::SlotId,
key: u64,
bit: u32,
out: *mut u32,
) {
for i in 0..5 {
unsafe { out.add(i).write(0) };
}
let Some((cell, _total, _wide, current, lo_base)) = slot_state_or_abort(handle, slot) else {
return;
};
let (forced, driven) =
crate::state::with_force_table(handle, |t| (t.contains(cell), t.driven_value(cell)));
let counts = crate::state::driver_counts_of(key, bit).unwrap_or_else(|| {
let v = driven.as_ref().unwrap_or(¤t);
match crate::state::four_state_bit_at(v, bit + lo_base) {
vj_abi::Bit4::Zero => [1, 0, 0],
vj_abi::Bit4::One => [0, 1, 0],
vj_abi::Bit4::X => [0, 0, 1],
vj_abi::Bit4::Z => [0, 0, 0],
}
});
if counts == crate::state::UNCOUNTED {
eprintln!(
"vj: runtime error: $countdrivers of a net in a bidirectional switch network is not supported"
);
real_sv_rt_finish(handle, 1);
return;
}
let all = [
u32::from(forced),
counts[0] + counts[1] + counts[2],
counts[0],
counts[1],
counts[2],
];
for (i, v) in all.into_iter().enumerate() {
unsafe { out.add(i).write(v) };
}
}
pub type PostponedThunk = extern "C" fn(SimHandle);
struct StrobeCallback {
thunk: PostponedThunk,
handle: SimHandle,
}
unsafe impl Send for StrobeCallback {}
impl super::RegionCallback for StrobeCallback {
fn invoke(&mut self, _ctx: &dyn super::SchedulerCtx) {
(self.thunk)(self.handle);
}
}
#[cfg_attr(
feature = "aot-exports",
unsafe(export_name = "sv_rt_strobe_postponed")
)]
pub extern "C" fn real_sv_rt_strobe(handle: SimHandle, thunk: PostponedThunk) {
with_current_ops(|ops| {
ops.register_region_callback(
vj_abi::Region::Postponed,
ops.now(),
Box::new(StrobeCallback { thunk, handle }),
);
});
}
thread_local! {
static MONITOR_GENERATION: std::cell::Cell<u64> = const { std::cell::Cell::new(0) };
static MONITOR_ENABLED: std::cell::Cell<bool> = const { std::cell::Cell::new(true) };
}
pub const MONITOR_DEDUPE_N: u32 = 1 << 31;
#[derive(Clone, Copy, PartialEq, Eq)]
enum MonitorMode {
Idle,
Probe,
Print,
}
struct MonitorDedupe {
mask: u32,
mode: MonitorMode,
last: Option<Vec<u8>>,
due: bool,
probe: Option<(PostponedThunk, SimHandle)>,
}
thread_local! {
static MONITOR_DEDUPE: std::cell::RefCell<Option<MonitorDedupe>> =
const { std::cell::RefCell::new(None) };
}
pub fn monitor_dedupe_reset(mask: Option<u32>) {
MONITOR_DEDUPE.with(|d| {
*d.borrow_mut() = mask.map(|mask| MonitorDedupe {
mask,
mode: MonitorMode::Idle,
last: None,
due: true,
probe: None,
});
});
}
fn monitor_dedupe_set_mode(mode: MonitorMode) {
MONITOR_DEDUPE.with(|d| {
if let Some(m) = d.borrow_mut().as_mut() {
m.mode = mode;
}
});
}
pub fn monitor_dedupe_arm(on: bool) {
monitor_dedupe_set_mode(if on {
MonitorMode::Print
} else {
MonitorMode::Idle
});
}
pub fn monitor_probe() {
let probe = MONITOR_DEDUPE.with(|d| {
d.borrow()
.as_ref()
.filter(|m| m.mode == MonitorMode::Idle)
.and_then(|m| m.probe)
});
if let Some((thunk, handle)) = probe {
monitor_dedupe_set_mode(MonitorMode::Probe);
thunk(handle);
monitor_dedupe_set_mode(MonitorMode::Idle);
}
}
fn monitor_dedupe_active() -> bool {
MONITOR_DEDUPE.with(|d| d.borrow().is_some())
}
fn monitor_dedupe_take_due() -> bool {
MONITOR_DEDUPE.with(|d| {
d.borrow_mut()
.as_mut()
.is_some_and(|m| std::mem::take(&mut m.due))
})
}
pub fn monitor_dedupe_check(make_key: impl FnOnce(u32) -> Vec<u8>) -> bool {
MONITOR_DEDUPE.with(|d| {
let mut d = d.borrow_mut();
let Some(m) = d.as_mut().filter(|m| m.mode != MonitorMode::Idle) else {
return false;
};
let key = make_key(m.mask);
let probing = m.mode == MonitorMode::Probe;
if probing && m.last.as_deref() != Some(key.as_slice()) {
m.due = true;
}
m.last = Some(key);
probing
})
}
struct MonitorCallback {
thunk: PostponedThunk,
handle: SimHandle,
generation: u64,
}
unsafe impl Send for MonitorCallback {}
impl super::RegionCallback for MonitorCallback {
fn invoke(&mut self, ctx: &dyn super::SchedulerCtx) {
if MONITOR_GENERATION.with(|g| g.get()) != self.generation {
return; }
if MONITOR_ENABLED.with(|e| e.get()) {
let changed = ctx.take_monitor_changed();
let dedupe = monitor_dedupe_active();
if (dedupe && monitor_dedupe_take_due()) || (!dedupe && changed) {
monitor_dedupe_arm(true);
(self.thunk)(self.handle);
monitor_dedupe_arm(false);
}
}
ctx.rearm(
vj_abi::Region::Postponed,
Box::new(MonitorCallback {
thunk: self.thunk,
handle: self.handle,
generation: self.generation,
}),
);
}
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_monitor_start"))]
pub unsafe extern "C" fn real_sv_rt_monitor(
handle: SimHandle,
thunk: PostponedThunk,
slots: *const u32,
n: u32,
) {
let generation = MONITOR_GENERATION.with(|g| {
let next = g.get() + 1;
g.set(next);
next
});
let dedupe = n & MONITOR_DEDUPE_N != 0;
let n = n & !MONITOR_DEDUPE_N;
let all: &[u32] = if n == 0 && !dedupe {
&[]
} else {
unsafe { std::slice::from_raw_parts(slots, n as usize + usize::from(dedupe)) }
};
let (slots, mask) = if dedupe {
(&all[..n as usize], Some(all[n as usize]))
} else {
(all, None)
};
monitor_dedupe_reset(mask);
if mask.is_some() {
MONITOR_DEDUPE.with(|d| {
if let Some(m) = d.borrow_mut().as_mut() {
m.probe = Some((thunk, handle));
}
});
}
let cells: Vec<StateCellId> = slots
.iter()
.map(|&s| crate::state::cell_for_slot(handle, vj_abi::SlotId(s)))
.collect();
with_current_ops(|ops| {
ops.set_monitor_watch(cells);
ops.register_region_callback(
vj_abi::Region::Postponed,
ops.now(),
Box::new(MonitorCallback {
thunk,
handle,
generation,
}),
);
});
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_monitor_enable"))]
pub extern "C" fn real_sv_rt_monitor_enable(_handle: SimHandle, on: i32) {
MONITOR_ENABLED.with(|e| e.set(on != 0));
if on != 0 {
MONITOR_DEDUPE.with(|d| {
if let Some(m) = d.borrow_mut().as_mut() {
m.due = true;
}
});
}
}
thread_local! {
static SVA_LEAF_TIME: std::cell::Cell<Option<u64>> = const { std::cell::Cell::new(None) };
}
pub fn with_sva_leaf_time<R>(ticks: u64, f: impl FnOnce() -> R) -> R {
let prev = SVA_LEAF_TIME.replace(Some(ticks));
let r = f();
SVA_LEAF_TIME.set(prev);
r
}
#[cfg_attr(feature = "aot-exports", unsafe(export_name = "sv_rt_time"))]
pub extern "C" fn real_sv_rt_time() -> u64 {
if let Some(t) = SVA_LEAF_TIME.get() {
return t;
}
let mut ticks: u64 = 0;
with_current_ops(|ops| ticks = ops.now().ticks);
ticks
}
pub const SV_KERNEL_ABI_RECONCILIATION: &str = "CG-084/CG-111..CG-116 (docs/design/codegen.md §9's SvKernelTable/SvProcessDesc/SvWakeSpec/\
SvResumeStatus literal draft) is superseded by ARCHITECTURE.md §5's frozen ABI \
(SimHandle/ProcessFn/EvalResult/Checkpoint), the same reconciliation PR-037/PR-107 already \
record for the process-model side. CompiledKernelProcess (vj-sim::sched::kernel_adapter) \
and lower_kernel_fn (vj-codegen::process_lower) implement the frozen shape; the old \
struct names are never constructed anywhere in this tree.";
#[cfg(test)]
mod tests {
use super::*;
use crate::sched::Region;
struct NoOpSchedulerOps;
impl SchedulerOps for NoOpSchedulerOps {
fn now(&self) -> crate::sched::SchedTime {
crate::sched::SchedTime { ticks: 0 }
}
fn current_region(&self) -> Region {
Region::Active
}
fn stage_nba(
&self,
_cell: StateCellId,
_value: crate::state::FourStateValue,
_source: u32,
) {
}
fn commit_blocking_write(&self, _cell: StateCellId, _value: crate::state::FourStateValue) {}
fn request_finish(&self, _code: i32) {}
fn defer_io(
&self,
_at: crate::sched::SchedTime,
_region: Region,
_cb: crate::sched::DeferredIoCallback,
) {
}
fn register_region_callback(
&self,
_region: Region,
_at: crate::sched::SchedTime,
_cb: Box<dyn crate::sched::RegionCallback>,
) {
}
}
extern "C" fn stub_kernel_initial_dispatch(
_h: SimHandle,
_local: *mut u8,
resume_pc: u32,
) -> EvalResult {
assert_eq!(
resume_pc, 0,
"PR-012: the first call to a freshly constructed CompiledKernelProcess must pass \
resume_pc == 0"
);
build_eval_result(ResolvedSuspend::Delay(5), 1)
}
#[test]
fn resume_at_pc_zero_is_initial_dispatch() {
let mut proc = CompiledKernelProcess::new(
stub_kernel_initial_dispatch,
0,
SimHandle(std::ptr::null_mut()),
);
assert_eq!(proc.resume_pc, 0);
let ops = NoOpSchedulerOps;
let suspend = proc.resume(SimHandle(std::ptr::null_mut()), ResumeArg::Start, &ops);
match suspend {
Suspend::Delay(t) => assert_eq!(t, 5),
other => panic!("expected Suspend::Delay(5), got {other:?}"),
}
assert_eq!(
proc.resume_pc, 1,
"the adapter must persist the decoded next resume_pc for the following call"
);
}
#[test]
fn resume_pc_round_trips_through_evalresult_and_reschedulereq() {
let er_edge = build_eval_result(
ResolvedSuspend::Edge {
cell: StateCellId(7),
edge: EdgeKind::Posedge,
},
3,
);
match decode_reschedule(er_edge).expect("KIND_EDGE must decode to Ok") {
Suspend::WaitEvents(set) => {
assert_eq!(set.terms.len(), 1);
match &set.terms[0] {
EventTerm::Edge { cell, edge, iff } => {
assert_eq!(*cell, StateCellId(7));
assert_eq!(*edge, EdgeKind::Posedge);
assert!(iff.is_none());
}
other => panic!("expected EventTerm::Edge, got {other:?}"),
}
}
other => panic!("expected Suspend::WaitEvents, got {other:?}"),
}
let er_delay = build_eval_result(ResolvedSuspend::Delay(10), 2);
match decode_reschedule(er_delay).expect("KIND_DELAY must decode to Ok") {
Suspend::Delay(t) => assert_eq!(t, 10),
other => panic!("expected Suspend::Delay(10), got {other:?}"),
}
let er_done = build_eval_result(ResolvedSuspend::Done, 0);
match decode_reschedule(er_done).expect("KIND_DONE must decode to Ok") {
Suspend::Done => {}
other => panic!("expected Suspend::Done, got {other:?}"),
}
}
#[test]
fn by_request_round_trips_through_reschedule_req() {
let er = build_eval_result(ResolvedSuspend::ByRequest, 9);
match decode_reschedule(er).expect("KIND_BY_REQUEST must decode to Ok") {
Suspend::ByRequest => {}
other => panic!("expected Suspend::ByRequest, got {other:?}"),
}
}
#[test]
fn repeat_edge_round_trips_through_reschedule_req_as_a_live_repeat_over_edge_term() {
let er = build_eval_result(
ResolvedSuspend::RepeatEdge {
count: 3,
cell: StateCellId(11),
edge: EdgeKind::Posedge,
},
4,
);
match decode_reschedule(er).expect("KIND_REPEAT_EDGE must decode to Ok") {
Suspend::WaitEvents(set) => {
assert_eq!(set.terms.len(), 1);
match &set.terms[0] {
EventTerm::Repeat { count, inner } => {
assert_eq!(*count, 3);
match inner.as_ref() {
EventTerm::Edge { cell, edge, iff } => {
assert_eq!(*cell, StateCellId(11));
assert_eq!(*edge, EdgeKind::Posedge);
assert!(iff.is_none());
}
other => panic!("expected EventTerm::Edge, got {other:?}"),
}
}
other => panic!("expected EventTerm::Repeat, got {other:?}"),
}
}
other => panic!("expected Suspend::WaitEvents, got {other:?}"),
}
}
#[test]
fn repeat_edge_count_round_trips_near_u64_max_through_the_wire() {
let near_max: u64 = u64::MAX - 1;
let er = build_eval_result(
ResolvedSuspend::RepeatEdge {
count: near_max,
cell: StateCellId(1),
edge: EdgeKind::Negedge,
},
0,
);
match decode_reschedule(er).expect("KIND_REPEAT_EDGE must decode to Ok") {
Suspend::WaitEvents(set) => match &set.terms[0] {
EventTerm::Repeat { count, .. } => assert_eq!(*count, near_max),
other => panic!("expected EventTerm::Repeat, got {other:?}"),
},
other => panic!("expected Suspend::WaitEvents, got {other:?}"),
}
}
#[test]
fn sc068_decode_reschedule_can_never_produce_a_some_iff_edge_term() {
let er = build_eval_result(
ResolvedSuspend::Edge {
cell: StateCellId(9),
edge: EdgeKind::Posedge,
},
0,
);
match decode_reschedule(er).expect("KIND_EDGE must decode to Ok") {
Suspend::WaitEvents(set) => {
assert_eq!(set.terms.len(), 1);
match &set.terms[0] {
EventTerm::Edge { iff, .. } => assert!(
iff.is_none(),
"SC-068 reachability gap regressed: decode_reschedule must never invent \
an iff predicate -- ResolvedSuspend::Edge (the only shape a real \
compiled kernel's resolve_suspend_point can produce) has no iff field \
to carry one across this ABI boundary today"
),
other => panic!("expected EventTerm::Edge, got {other:?}"),
}
}
other => panic!("expected Suspend::WaitEvents, got {other:?}"),
}
}
extern "C" fn stub_guard_fn(_h: SimHandle) -> bool {
false
}
#[test]
fn guard_fn_round_trips_through_reschedule_req() {
let er = build_eval_result(
ResolvedSuspend::EdgeWithGuard {
cell: StateCellId(11),
edge: EdgeKind::Negedge,
guard_fn: stub_guard_fn,
},
3,
);
match decode_reschedule(er).expect("KIND_EDGE (with guard) must decode to Ok") {
Suspend::WaitEvents(set) => {
assert_eq!(set.terms.len(), 1);
match &set.terms[0] {
EventTerm::Edge { cell, edge, iff } => {
assert_eq!(*cell, StateCellId(11));
assert_eq!(*edge, EdgeKind::Negedge);
let PredicateFn(decoded) =
iff.expect("an EdgeWithGuard suspend must decode iff: Some(..)");
assert_eq!(
decoded as *const () as usize, stub_guard_fn as *const () as usize,
"the decoded guard function pointer must be byte-identical to the \
original"
);
}
other => panic!("expected EventTerm::Edge, got {other:?}"),
}
}
other => panic!("expected Suspend::WaitEvents, got {other:?}"),
}
}
#[test]
fn decode_reschedule_reports_typed_errors_for_edge_list_and_fork_join() {
use crate::sched::JoinKind;
fn wrap(proposal: RescheduleReqLayoutProposal) -> EvalResult {
let mut opaque = [0u8; vj_abi::eval::RESCHEDULE_REQ_SIZE];
opaque[..PROPOSAL_SIZE].copy_from_slice(&proposal.to_bytes());
EvalResult {
outcome: EvalOutcome::SuspendedOnEvent,
reschedule: RescheduleReq { opaque },
}
}
let edge_list = wrap(RescheduleReqLayoutProposal::from_edge_list(ArenaRange {
base: 3,
count: 5,
}));
match decode_reschedule(edge_list) {
Err(DecodeRescheduleError::UnresolvedEdgeList { base, count }) => {
assert_eq!(base, 3);
assert_eq!(count, 5);
}
other => panic!(
"KIND_EDGE_LIST must decode to Err(UnresolvedEdgeList{{3,5}}), never panic; got \
{other:?}"
),
}
let fork_join = wrap(RescheduleReqLayoutProposal::from_fork_join(
JoinKind::All,
ArenaRange { base: 10, count: 2 },
));
match decode_reschedule(fork_join) {
Err(DecodeRescheduleError::UnresolvedForkJoin {
join_kind,
base,
count,
}) => {
assert_eq!(join_kind, JoinKind::All as u8);
assert_eq!(base, 10);
assert_eq!(count, 2);
}
other => panic!(
"KIND_FORK_JOIN must decode to Err(UnresolvedForkJoin{{..}}), never panic; got \
{other:?}"
),
}
}
#[test]
fn decode_reschedule_reports_a_typed_error_for_an_out_of_range_kind_byte() {
let mut opaque = [0u8; vj_abi::eval::RESCHEDULE_REQ_SIZE];
opaque[0] = 200;
let er = EvalResult {
outcome: EvalOutcome::Completed,
reschedule: RescheduleReq { opaque },
};
match decode_reschedule(er) {
Err(DecodeRescheduleError::InvalidKind(200)) => {}
other => panic!("expected Err(InvalidKind(200)), never a panic; got {other:?}"),
}
}
#[test]
fn sv_kernel_abi_reconciliation_note_exists_and_is_referenced() {
assert!(
SV_KERNEL_ABI_RECONCILIATION.contains("CG-084"),
"the reconciliation note must cite CG-084 so a reader auditing that ledger row \
finds this exact note"
);
}
#[test]
fn exactly_one_compiledprocess_impl_over_a_kernel_backed_type() {
const UNRELATED_PLACEHOLDER: &str = "impl CompiledProcess for UnmintedForkChild";
let src_root = std::path::Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/src"));
let mut total = 0usize;
let mut stack = vec![src_root.to_path_buf()];
while let Some(dir) = stack.pop() {
for entry in std::fs::read_dir(&dir).expect("read_dir must succeed under src/") {
let entry = entry.expect("dir entry must be readable");
let path = entry.path();
if path.is_dir() {
stack.push(path);
continue;
}
if path.extension().and_then(|e| e.to_str()) != Some("rs") {
continue;
}
let text = std::fs::read_to_string(&path)
.unwrap_or_else(|e| panic!("must be able to read {path:?}: {e}"));
let production_only =
crate::determinism::scan_support::strip_cfg_test_blocks(&text);
let here = production_only.matches("impl CompiledProcess for").count()
- production_only.matches(UNRELATED_PLACEHOLDER).count();
total += here;
}
}
assert_eq!(
total, 1,
"expected exactly one kernel-backed production `impl CompiledProcess for` in the \
whole crate (this module's own, on CompiledKernelProcess) -- found {total}"
);
}
struct RecordingSchedulerOps {
finish_calls: std::cell::RefCell<Vec<i32>>,
}
impl SchedulerOps for RecordingSchedulerOps {
fn now(&self) -> crate::sched::SchedTime {
crate::sched::SchedTime { ticks: 0 }
}
fn current_region(&self) -> Region {
Region::Active
}
fn stage_nba(
&self,
_cell: StateCellId,
_value: crate::state::FourStateValue,
_source: u32,
) {
}
fn commit_blocking_write(&self, _cell: StateCellId, _value: crate::state::FourStateValue) {}
fn request_finish(&self, code: i32) {
self.finish_calls.borrow_mut().push(code);
}
fn defer_io(
&self,
_at: crate::sched::SchedTime,
_region: Region,
_cb: crate::sched::DeferredIoCallback,
) {
}
fn register_region_callback(
&self,
_region: Region,
_at: crate::sched::SchedTime,
_cb: Box<dyn crate::sched::RegionCallback>,
) {
}
}
#[test]
fn with_scheduler_ops_makes_ops_reachable_to_real_sv_rt_finish() {
let ops = RecordingSchedulerOps {
finish_calls: std::cell::RefCell::new(Vec::new()),
};
let handle = SimHandle(std::ptr::null_mut());
with_scheduler_ops(&ops, || {
real_sv_rt_finish(handle, 7);
});
assert_eq!(
*ops.finish_calls.borrow(),
vec![7],
"real_sv_rt_finish must reach the real ops.request_finish while inside \
with_scheduler_ops's callback"
);
}
#[test]
fn real_sv_rt_finish_outside_with_scheduler_ops_does_not_panic() {
let handle = SimHandle(std::ptr::null_mut());
real_sv_rt_finish(handle, 1);
}
#[test]
fn with_current_process_makes_proc_reachable_for_exactly_one_call_and_restores_after() {
assert_eq!(
CURRENT_PROCESS.with(|c| *c.borrow()),
None,
"precondition: no other test on this thread left CURRENT_PROCESS set"
);
let seen_inside = with_current_process(42, || CURRENT_PROCESS.with(|c| *c.borrow()));
assert_eq!(
seen_inside,
Some(42),
"CURRENT_PROCESS must read back the ProcessId with_current_process installed, \
from inside the wrapped call"
);
assert_eq!(
CURRENT_PROCESS.with(|c| *c.borrow()),
None,
"with_current_process must restore the prior (None) value after the wrapped call \
returns, not leave 42 installed"
);
}
#[test]
fn with_current_process_restores_the_outer_value_not_merely_clears_it() {
with_current_process(1, || {
let inner = with_current_process(2, || CURRENT_PROCESS.with(|c| *c.borrow()));
assert_eq!(inner, Some(2));
assert_eq!(
CURRENT_PROCESS.with(|c| *c.borrow()),
Some(1),
"returning from a nested with_current_process must restore the OUTER caller's \
ProcessId, not clear it to None"
);
});
assert_eq!(
CURRENT_PROCESS.with(|c| *c.borrow()),
None,
"after the outermost with_current_process call returns, CURRENT_PROCESS must be \
back to None"
);
}
#[test]
fn real_nba_enqueue_prefers_current_process_over_the_raw_abi_source() {
let template = crate::netlist::TemplateKey {
definition: 1,
params_hash: 0,
};
let layout = crate::state::StateLayout {
slots: vec![crate::state::SlotDescriptor {
key: crate::netlist::SymbolKey(0),
offset: crate::state::SlotOffset {
class: crate::state::SlotClass::Inline,
word_offset: 0,
bit_width: 1,
elem_word_count: 1,
row_stride_words: 0,
num_rows: 0,
},
initial: None,
net_info: None,
}],
inline_words: 1,
wide_words: 0,
};
let registry = crate::state::TemplateRegistry::new();
unsafe {
registry.register(template, layout);
}
let mut engine = crate::state::StateEngine::new(registry);
let handle = engine
.allocate_instance(template)
.expect("template was just registered above");
let cell = handle.cell_for_slot(0);
let h = handle.into_sim_handle();
struct NbaRecordingOps {
calls: std::cell::RefCell<Vec<(StateCellId, u32)>>,
}
impl SchedulerOps for NbaRecordingOps {
fn now(&self) -> crate::sched::SchedTime {
crate::sched::SchedTime { ticks: 0 }
}
fn current_region(&self) -> Region {
Region::Nba
}
fn stage_nba(
&self,
cell: StateCellId,
_value: crate::state::FourStateValue,
source: u32,
) {
self.calls.borrow_mut().push((cell, source));
}
fn commit_blocking_write(
&self,
_cell: StateCellId,
_value: crate::state::FourStateValue,
) {
}
fn request_finish(&self, _code: i32) {}
fn defer_io(
&self,
_at: crate::sched::SchedTime,
_region: Region,
_cb: crate::sched::DeferredIoCallback,
) {
}
fn register_region_callback(
&self,
_region: Region,
_at: crate::sched::SchedTime,
_cb: Box<dyn crate::sched::RegionCallback>,
) {
}
}
let ops = NbaRecordingOps {
calls: std::cell::RefCell::new(Vec::new()),
};
with_scheduler_ops(&ops, || {
real_nba_enqueue(h, vj_abi::SlotId(0), 1, 99);
});
assert_eq!(
*ops.calls.borrow(),
vec![(cell, 99)],
"with no CURRENT_PROCESS installed, real_nba_enqueue must fall back to the raw ABI \
source value"
);
ops.calls.borrow_mut().clear();
with_current_process(7, || {
with_scheduler_ops(&ops, || {
real_nba_enqueue(h, vj_abi::SlotId(0), 1, 99);
});
});
assert_eq!(
*ops.calls.borrow(),
vec![(cell, 7)],
"with CURRENT_PROCESS installed, real_nba_enqueue must prefer it (7) over the raw \
ABI source (99)"
);
unsafe {
drop(Box::from_raw(h.0 as *mut InstanceHandle));
}
}
}