use std::borrow::Cow;
use super::library::{Ctx, Next};
use super::*;
use crate::error::CompileErrorKind;
use crate::heap::{RESERVED_NAMES, Task};
use crate::host::Builtin;
const READER_CHUNKNAME: &[u8] = b"=(load)";
pub(super) enum LoadSource {
Values(Vec<Value>),
}
#[inline(never)]
fn fast_builtin_call(
thread: &mut ThreadObj,
heap: &mut FrameHeap<'_>,
function: Value,
args: &[Value],
slot: u32,
wants: u8,
) -> bool {
let Value::Closure(closure) = function else {
return false;
};
let Some(proto) = heap
.closures
.get(closure)
.and_then(|closure| heap.protos.get(closure.proto))
else {
return false;
};
if proto.vararg {
return false;
}
let base = slot + 1;
let scratch_end = base + args.len() as u32;
let len = thread.stack.len() as u32;
let scratch_charge =
cost::STACK_SLOT * u64::from(scratch_end.saturating_sub(thread.charged_slots));
if scratch_end > heap.ordinary
|| !heap
.gc
.fits(cost::STACK_SLOT * u64::from(scratch_end.saturating_sub(len)))
{
return false;
}
let Some(admit) = super::check_fixed_frame(
thread.frames.depth,
len.max(scratch_end) as usize,
heap,
proto,
base,
args.len() as u32,
Some(scratch_charge),
) else {
return false;
};
if scratch_end > len {
thread.charge_slots(scratch_end as usize, heap.gc);
#[cfg(feature = "counters")]
for _ in len..scratch_end {
count!("stack_growths");
}
thread.stack.grow_to(scratch_end as usize);
}
thread.stack[slot as usize] = function;
thread.stack[base as usize..scratch_end as usize].copy_from_slice(args);
super::write_fixed_frame(
thread,
heap,
admit,
closure,
proto.params,
base,
args.len() as u32,
wants,
false,
);
true
}
#[inline(never)]
pub(super) fn fast_builtin_return(thread: &mut ThreadObj, base: u8, count: u8) -> FrameStep {
if thread.frames.len() < 2 {
return FrameStep::Cold;
}
let frame = thread.frames.last().expect("callback callee");
let caller = &thread.frames[thread.frames.len() - 2];
if !matches!(
caller.boundary(),
Some(Boundary::Builtin { .. } | Boundary::Handler { .. })
) || caller.pending().is_some()
|| caller.meta().is_some()
|| thread.open_above > frame.base
|| thread.tbc.last().is_some_and(|slot| *slot >= frame.base)
{
return FrameStep::Cold;
}
if frame.nresults == 1
&& count == 1
&& matches!(
caller.boundary(),
Some(Boundary::Builtin {
task: Task::Lib(task),
..
}) if matches!(task.wait, crate::library::Wait::Truth)
)
{
let dest = frame.base.saturating_sub(frame.vararg_len + 1) as usize;
let src = frame.base as usize + usize::from(base);
let clear_from = dest + 1;
let clear_end = (frame.limit as usize).max(clear_from);
if src >= thread.stack.len()
|| clear_from > thread.stack.len()
|| clear_end > thread.stack.len()
{
return FrameStep::Cold;
}
let result = thread.stack[src];
thread.frames.pop();
thread.stack[dest] = result;
thread.stack[clear_from..clear_end].fill(Value::Nil);
let limit = thread
.frames
.iter()
.rev()
.find(|frame| frame.boundary().is_none())
.map_or(0, |frame| frame.limit as usize);
thread.stack.truncate(limit.max(clear_from));
thread.top = clear_from as u32;
return FrameStep::Continue;
}
let dest = frame.base.saturating_sub(frame.vararg_len + 1) as usize;
let src = frame.base as usize + usize::from(base);
let produced = if count == COUNT_OPEN {
(thread.top as usize).saturating_sub(src)
} else {
usize::from(count)
};
let want = if frame.nresults == COUNT_OPEN {
produced
} else {
usize::from(frame.nresults)
};
let clear_from = dest + if want == 0 { 1 } else { want };
let clear_end = (frame.limit as usize).max(clear_from);
if dest + want > thread.stack.len() || clear_end > thread.stack.len() {
return FrameStep::Cold;
}
let copied = want.min(produced);
if src + copied > thread.stack.len() {
return FrameStep::Cold;
}
if copied != 0 {
thread.stack.copy_within(src..src + copied, dest);
}
thread.stack[dest + copied..dest + want].fill(Value::Nil);
thread.frames.pop();
thread.stack[clear_from..clear_end].fill(Value::Nil);
let limit = thread
.frames
.iter()
.rev()
.find(|frame| frame.boundary().is_none())
.map_or(0, |frame| frame.limit as usize);
let new_top = dest + want;
thread.stack.truncate(limit.max(new_top));
thread.top = new_top as u32;
FrameStep::Continue
}
pub(super) fn type_index(value: Value) -> usize {
match value {
Value::Nil => 0,
Value::Bool(_) => 1,
Value::Integer(_) | Value::Float(_) => 2,
Value::String(_) => 3,
Value::Table(_) => 4,
Value::Closure(_) | Value::Native(_) | Value::NativeClosure(_) => 5,
Value::Thread(_) => 6,
Value::Userdata(_) | Value::LightUserdata(..) => 7,
}
}
pub(crate) fn plain_text(heap: &Heap, value: Value) -> Cow<'_, [u8]> {
match value {
Value::String(handle) => Cow::Borrowed(heap.string_bytes(handle).unwrap_or_default()),
Value::Nil => Cow::Borrowed(b"nil"),
Value::Bool(true) => Cow::Borrowed(b"true"),
Value::Bool(false) => Cow::Borrowed(b"false"),
Value::Integer(_) | Value::Float(_) => Cow::Owned(
crate::concat::number_text(value)
.unwrap_or_default()
.into_bytes(),
),
Value::Native(index) => {
let symbol = heap.natives.get(index as usize).map_or("?", String::as_str);
Cow::Owned(format!("function: builtin: {symbol}").into_bytes())
}
Value::Table(_)
| Value::Closure(_)
| Value::Thread(_)
| Value::NativeClosure(_)
| Value::Userdata(_)
| Value::LightUserdata(..) => {
let named = match index::metamethod(heap, value, b"__name") {
Some(Value::String(handle)) => heap.string_bytes(handle),
_ => None,
};
let mut text = match named {
Some(name) => name.to_vec(),
None => RESERVED_NAMES[type_index(value)].as_bytes().to_vec(),
};
text.extend_from_slice(b": ");
text.extend_from_slice(&identity_text(heap, value));
Cow::Owned(text)
}
}
}
pub(crate) fn identity_text(heap: &Heap, value: Value) -> Vec<u8> {
match value {
Value::LightUserdata(crate::value::LightDomain::Host, bits) => {
format!("0x{bits:016x}").into_bytes()
}
Value::LightUserdata(_, bits) => format!("0x{bits:08x}").into_bytes(),
_ => {
let id = heap.object_id_of_value(value).map_or(0, ObjectId::raw);
format!("0x{id:08x}").into_bytes()
}
}
}
pub(super) fn text_arg(heap: &Heap, value: Value) -> Option<Cow<'_, [u8]>> {
match value {
Value::String(handle) => heap.string_bytes(handle).map(Cow::Borrowed),
Value::Integer(_) | Value::Float(_) => Some(plain_text(heap, value)),
_ => None,
}
}
fn integer_in_base(bytes: &[u8], base: u32) -> Option<i64> {
let space = |byte: &u8| matches!(byte, b' ' | b'\x0c' | b'\n' | b'\r' | b'\t' | b'\x0b');
let start = bytes.iter().position(|byte| !space(byte))?;
let mut text = &bytes[start..];
let negative = match text.first() {
Some(b'-') => {
text = &text[1..];
true
}
Some(b'+') => {
text = &text[1..];
false
}
_ => false,
};
let digits = text
.iter()
.position(|byte| !byte.is_ascii_alphanumeric())
.unwrap_or(text.len());
if digits == 0 || !text[digits..].iter().all(space) {
return None;
}
let mut value: u64 = 0;
for byte in &text[..digits] {
let digit = (*byte as char).to_digit(36)?;
if digit >= base {
return None;
}
value = value
.wrapping_mul(u64::from(base))
.wrapping_add(u64::from(digit));
}
let value = value as i64;
Some(if negative {
value.wrapping_neg()
} else {
value
})
}
impl Runtime {
pub(super) fn call_base(
&mut self,
builtin: Builtin,
journal: &mut Journal,
) -> Result<Poll, VmError> {
let active = self.heap.active.ok_or(VmError::Corrupt)?;
let site @ (func, _, passed, _) = self.call_site(active)?;
let arg = |runtime: &Self, index: u32| -> Value {
if index < passed {
runtime
.heap
.threads
.get(active)
.and_then(|thread| thread.stack.get((func + 1 + index) as usize))
.copied()
.unwrap_or(Value::Nil)
} else {
Value::Nil
}
};
let present = |index: u32| index < passed && !matches!(arg(self, index), Value::Nil);
match builtin {
Builtin::Assert => {
if passed == 0 {
return self.base_error(active, |r, c| r.bad_arg(c, 0, "value expected"));
}
if arg(self, 0).truthy() {
let mut values = std::mem::take(&mut self.native_results);
values.clear();
values.extend((0..passed).map(|index| arg(self, index)));
let result = self.base_return(active, &values);
values.clear();
self.native_results = values;
return result;
}
if passed >= 2 {
let mut message = arg(self, 1);
if let Value::String(handle) = message {
let text = self.heap.string_bytes(handle).unwrap_or_default().to_vec();
let location = self.location(active, 1).filter(|(_, line)| *line > 0);
message = self.prefixed(location, text, LuaFault::Error);
}
return Ok(self.throw_on(active, LuaFault::Error, message));
}
Ok(self.fault(LuaFault::Assert))
}
Builtin::Type => {
if passed == 0 {
return self.base_error(active, |r, c| r.bad_arg(c, 0, "value expected"));
}
let name = self.reserved_name(type_index(arg(self, 0)))?;
self.base_return(active, &[name])
}
Builtin::ToNumber => {
let value = arg(self, 0);
if !present(1) {
if passed == 0 {
return self.base_error(active, |r, c| r.bad_arg(c, 0, "value expected"));
}
let number = match value {
Value::Integer(_) | Value::Float(_) => value,
Value::String(handle) => self
.heap
.string_bytes(handle)
.and_then(crate::lex::string_to_number)
.unwrap_or(Value::Nil),
_ => Value::Nil,
};
return self.base_return(active, &[number]);
}
let Some(base) = crate::base::lua_integer(&self.heap, arg(self, 1)) else {
return self.base_error(active, |r, c| {
r.int_arg(c, 1)
.err()
.unwrap_or(Next::Fault(LuaFault::Argument))
});
};
let Value::String(handle) = value else {
return self.base_error(active, |r, c| r.bad_type(c, 0, "string"));
};
if !(2..=36).contains(&base) {
return self.base_error(active, |r, c| r.bad_arg(c, 1, "base out of range"));
}
let number = self
.heap
.string_bytes(handle)
.and_then(|bytes| integer_in_base(bytes, base as u32))
.map_or(Value::Nil, Value::Integer);
self.base_return(active, &[number])
}
Builtin::ToString => {
if passed == 0 {
return self.base_error(active, |r, c| r.bad_arg(c, 0, "value expected"));
}
let value = arg(self, 0);
if let Some(method) = index::metamethod(&self.heap, value, b"__tostring") {
self.push_builtin(active, Task::ToString)?;
return self.builtin_call(method, &[value], journal);
}
let text = self.text_value(value)?;
self.base_return(active, &[text])
}
Builtin::Os(function) => self.call_os(active, function, journal),
Builtin::Exit => {
let status = match arg(self, 0) {
Value::Nil | Value::Bool(true) => crate::ExitStatus::Success,
Value::Bool(false) => crate::ExitStatus::Failure,
value => {
let Some(code) = crate::base::lua_integer(&self.heap, value) else {
return self.base_error(active, |r, c| {
r.int_arg(c, 0)
.err()
.unwrap_or(Next::Fault(LuaFault::Argument))
});
};
crate::ExitStatus::Code(code as i32)
}
};
self.request_exit(status, arg(self, 1).truthy())
}
Builtin::Print => self.print_from(active, 0, 0, false, journal),
Builtin::Warn => {
for index in 0..passed.max(1) {
if index >= passed || text_arg(&self.heap, arg(self, index)).is_none() {
return self.base_error(active, |r, c| r.bad_type(c, index, "string"));
}
}
let args: Vec<Value> = (0..passed).map(|index| arg(self, index)).collect();
self.warn_effect(journal, |heap, piece| {
for value in &args {
piece(&text_arg(heap, *value).unwrap_or_default());
}
});
self.base_return(active, &[])
}
Builtin::Next => {
let Value::Table(table) = arg(self, 0) else {
return self.base_error(active, |r, c| r.bad_type(c, 0, "table"));
};
let key = arg(self, 1);
if self.iteration_profile == IterationProfile::Sorted {
return match self.sorted_next(table, key)? {
Ok(Some((key, value))) => self.base_return(active, &[key, value]),
Ok(None) => self.base_return(active, &[Value::Nil]),
Err(error) => Ok(self.sorted_next_fault(error)),
};
}
let normalized = if matches!(key, Value::Nil) {
None
} else {
match self.heap.key_view(key) {
Ok(key) => Some(key),
Err(LuaFault::NanKey | LuaFault::NilKey) => {
return Ok(self.fault(LuaFault::NextKey));
}
Err(fault) => return Ok(self.fault(fault)),
}
};
let found = self
.heap
.tables
.get(table)
.ok_or(VmError::Corrupt)?
.table
.next_view(normalized);
match found {
Ok(Some((key, value))) => self.base_return(active, &[key, value]),
Ok(None) => self.base_return(active, &[Value::Nil]),
Err(()) => Ok(self.fault(LuaFault::NextKey)),
}
}
Builtin::Pairs => {
if passed == 0 {
return self.base_error(active, |r, c| r.bad_arg(c, 0, "value expected"));
}
let table = arg(self, 0);
if let Some(method) = index::metamethod(&self.heap, table, b"__pairs") {
self.push_builtin(active, Task::Pairs)?;
return self.builtin_call(method, &[table], journal);
}
let next = self.native_value("base.next")?;
self.base_return(active, &[next, table, Value::Nil])
}
Builtin::Ipairs => {
if passed == 0 {
return self.base_error(active, |r, c| r.bad_arg(c, 0, "value expected"));
}
let iterator = self.native_value(crate::base::IPAIRS_NEXT)?;
self.base_return(active, &[iterator, arg(self, 0), Value::Integer(0)])
}
Builtin::IpairsNext => {
let Some(index) = crate::base::lua_integer(&self.heap, arg(self, 1)) else {
return self.base_error(active, |r, c| {
r.int_arg(c, 1)
.err()
.unwrap_or(Next::Fault(LuaFault::Argument))
});
};
let index = index.wrapping_add(1);
let obj = arg(self, 0);
match index::get(&self.heap, obj, Value::Integer(index)) {
Err(fault) => {
let op = library::Op::Get {
obj,
key: Value::Integer(index),
into: 0,
};
if let Some(text) = self.library_op_message(fault, op) {
let error = self.prefixed(None, text, fault);
Ok(self.throw_on(active, fault, error))
} else {
Ok(self.fault(fault))
}
}
Ok(Resolved::Done(value)) => {
self.base_return(active, &ipairs_step(index, value))
}
Ok(Resolved::Call { function, target }) => {
self.push_builtin(active, Task::Ipairs { index })?;
self.builtin_call(function, &[target, Value::Integer(index)], journal)
}
}
}
Builtin::CollectGarbage => self.collect_garbage(active, arg(self, 0)),
Builtin::Load => self.load_call(active, passed, journal),
Builtin::LoadFile | Builtin::DoFile => {
self.start_file(active, builtin == Builtin::DoFile, journal)
}
Builtin::Math(function) => self.call_math(active, function, journal),
Builtin::Table(function) => self.call_table(active, function, journal),
Builtin::String(function) => self.call_string(active, function, journal),
Builtin::Io(function) => self.call_io(active, function, journal),
Builtin::Utf8(function) => self.call_utf8(active, function, journal),
Builtin::Package(function) => self.call_package(active, function, journal),
Builtin::Debug(function) => self.call_debug(active, function, journal),
Builtin::Coroutine(function) => self.call_coroutine(active, function, site),
#[cfg(test)]
Builtin::CapabilitySmoke => Err(VmError::Corrupt),
Builtin::Pcall | Builtin::Xpcall | Builtin::Error => Err(VmError::Corrupt),
}
}
fn base_error(
&mut self,
active: Handle<ThreadObj>,
make: impl FnOnce(&Self, &Ctx) -> Next,
) -> Result<Poll, VmError> {
let (func, _, passed, _) = self.call_site(active)?;
let ctx = Ctx {
active,
func,
passed,
framed: false,
};
let next = make(self, &ctx);
Ok(self.finish_next(active, next))
}
pub(super) fn reserved_name(&self, index: usize) -> Result<Value, VmError> {
self.heap
.reserved
.get(LuaFault::ALL.len() + index)
.map(|handle| Value::String(*handle))
.ok_or(VmError::Corrupt)
}
fn text_value(&mut self, value: Value) -> Result<Value, VmError> {
match value {
Value::String(_) => Ok(value),
Value::Nil => self.reserved_name(0),
Value::Bool(true) => self.reserved_name(8),
Value::Bool(false) => self.reserved_name(9),
_ => {
let text = plain_text(&self.heap, value).into_owned();
self.new_string(text)
}
}
}
pub(super) fn new_string(&mut self, bytes: Vec<u8>) -> Result<Value, VmError> {
self.make_room(1, cost::OBJECT + bytes.len() as u64);
Ok(Value::String(self.alloc_string(bytes)?))
}
pub(super) fn base_return(
&mut self,
active: Handle<ThreadObj>,
values: &[Value],
) -> Result<Poll, VmError> {
let (func, nresults, passed, _) = self.call_site(active)?;
self.native_returned(active, func, nresults, passed, values)
}
pub(super) fn push_builtin(
&mut self,
active: Handle<ThreadObj>,
task: Task,
) -> Result<(), VmError> {
let (func, nresults, passed, _) = self.call_site(active)?;
let object = self.heap.threads.get_mut(active).ok_or(VmError::Corrupt)?;
let caller = object.frames.last().ok_or(VmError::Corrupt)?;
let advance_caller = caller.boundary().is_none() && caller.meta().is_none();
let closure = caller.closure;
count!("frame_pushes");
object.frames.push(Frame {
closure,
pc: 0,
base: func + 1,
limit: func + 1,
nresults,
vararg_len: 0,
flags: 0,
cold: FrameCold::with_boundary(
Boundary::Builtin {
func,
passed,
advance_caller,
task,
},
&mut self.cold_spare,
),
});
Ok(())
}
pub(super) fn builtin_slot(&self, active: Handle<ThreadObj>) -> Result<(u32, u8), VmError> {
let frame = self
.heap
.threads
.get(active)
.and_then(|thread| thread.frames.last())
.ok_or(VmError::Corrupt)?;
match frame.boundary() {
Some(Boundary::Builtin {
func, passed, task, ..
}) => Ok((func + 1 + passed + task.scratch(), task.wants())),
_ => Err(VmError::Corrupt),
}
}
pub(super) fn builtin_call(
&mut self,
function: Value,
args: &[Value],
journal: &mut Journal,
) -> Result<Poll, VmError> {
let active = self.heap.active.ok_or(VmError::Corrupt)?;
let slot = self.builtin_slot(active)?;
self.builtin_call_at(active, function, args, slot, journal)
}
pub(super) fn builtin_call_at(
&mut self,
active: Handle<ThreadObj>,
function: Value,
args: &[Value],
(slot, wants): (u32, u8),
journal: &mut Journal,
) -> Result<Poll, VmError> {
if !self.hook_trap
&& matches!(function, Value::Closure(_))
&& self.try_fast_boundary_call(active, function, args, slot, wants)?
{
return Ok(Poll::Continue);
}
if let Some(fault) = self.depth_fault(active)? {
return Ok(self.fault(fault));
}
if let Some(fault) = self.slot_fault(active, slot + 1 + args.len() as u32)? {
return Ok(self.fault(fault));
}
self.write_abs(active, slot, function)?;
for (offset, arg) in args.iter().enumerate() {
self.write_abs(active, slot + 1 + offset as u32, *arg)?;
}
let (function, nargs) = match self.resolve_callable(active, slot, args.len() as u32)? {
Ok(resolved) => resolved,
Err(fault) => {
let thread = self.heap.threads.get_mut(active).ok_or(VmError::Corrupt)?;
thread.stack.truncate(slot as usize);
thread.top = slot;
return Ok(self.fault(fault));
}
};
self.heap
.threads
.get_mut(active)
.ok_or(VmError::Corrupt)?
.top = slot + 1 + nargs;
match self.callable(function) {
Value::Closure(closure) => self.push_lua_frame(closure, slot, nargs, wants, false),
Value::Native(index) if self.is_builtin(index)? => self.defer_call(active),
Value::Native(index) => self.call_native(index, journal),
_ => Err(VmError::Corrupt),
}
}
pub(super) fn try_fast_boundary_call(
&mut self,
active: Handle<ThreadObj>,
function: Value,
args: &[Value],
slot: u32,
wants: u8,
) -> Result<bool, VmError> {
#[cfg(any(test, debug_assertions))]
let fast = self.hot_core == HotCoreMode::Full;
#[cfg(not(any(test, debug_assertions)))]
let fast = true;
if fast && matches!(function, Value::Closure(_)) {
let thread = self.heap.threads.get_mut(active).ok_or(VmError::Corrupt)?;
let mut heap = FrameHeap {
closures: &self.heap.closures,
protos: &self.heap.protos,
gc: &mut self.heap.gc,
ordinary: self.max_stack_slots - self.max_stack_slots / 8,
stack_grows: &mut self.stack_grows,
frame_grows: &mut self.frame_grows,
cold_spare: &mut self.cold_spare,
};
if fast_builtin_call(thread, &mut heap, function, args, slot, wants) {
return Ok(true);
}
}
Ok(false)
}
#[inline(never)]
pub(super) fn finish_hot_boundary(
&mut self,
quantum: &mut u64,
journal: &mut Journal,
) -> Result<Option<Poll>, VmError> {
#[cfg(any(test, debug_assertions))]
if self.hot_core != HotCoreMode::Full {
return Ok(None);
}
let active = self.heap.active.ok_or(VmError::Corrupt)?;
let Some(ready @ (Ready::Builtin | Ready::Handler)) = self.boundary_ready(active) else {
return Ok(None);
};
if *quantum == 0
|| self.trap.is_some()
|| self.fuel_consumed >= self.fuel_limit.unwrap_or(u64::MAX)
{
return Ok(None);
}
let thread = self.heap.threads.get(active).ok_or(VmError::Corrupt)?;
if thread.status != Status::Ready
|| thread.unwind.is_some()
|| thread
.frames
.last()
.is_some_and(|frame| frame.meta().is_some())
{
return Ok(None);
}
self.gc_schedule();
if self.gc_wanted() || self.finalizer_due(active) {
return Ok(None);
}
let step = match ready {
Ready::Builtin => {
self.fuel_consumed += 1;
*quantum -= 1;
self.finish_builtin(active, journal)
}
Ready::Handler => self.finish_handler(active),
_ => Err(VmError::Corrupt),
}
.or_else(|error| self.vm_error(error))?;
Ok(Some(step))
}
pub(super) fn builtin_done(
&mut self,
active: Handle<ThreadObj>,
values: &[Value],
) -> Result<Poll, VmError> {
let object = self.heap.threads.get_mut(active).ok_or(VmError::Corrupt)?;
let scratch_end = u32::try_from(object.stack.len())
.map_err(|_| VmError::Corrupt)?
.max(object.top);
let mut frame = object.frames.pop().ok_or(VmError::Corrupt)?;
let Some(Boundary::Builtin {
func,
advance_caller,
task,
passed,
}) = frame.cold.as_mut().and_then(|cold| cold.boundary.take())
else {
return Err(VmError::Corrupt);
};
let nresults = frame.nresults;
if let Task::Lib(mut task) = task {
*task = crate::library::LibTask {
work: crate::library::Work::Extreme {
max: false,
best: 0,
next: 0,
},
wait: crate::library::Wait::Nothing,
};
self.lib_task_spare = Some(task);
}
frame.recycle_cold(&mut self.cold_spare);
let produced = u32::try_from(values.len()).map_err(|_| VmError::Corrupt)?;
if let Some(fault) = self.slot_fault(active, func + Self::wanted(nresults, produced))? {
return Ok(self.fault(fault));
}
if self.hook_trap
&& self.capture_hook_native_return(active, func, nresults, passed, values)?
{
return Ok(Poll::Continue);
}
self.deliver_native(active, func, nresults, values, scratch_end)?;
if advance_caller {
self.bump_pc(active)?;
}
Ok(Poll::Continue)
}
pub(super) fn finish_builtin(
&mut self,
active: Handle<ThreadObj>,
journal: &mut Journal,
) -> Result<Poll, VmError> {
count!("builtin_resume_steps");
let object = self.heap.threads.get_mut(active).ok_or(VmError::Corrupt)?;
let (slot, func, passed) = match object.frames.last().and_then(|frame| frame.boundary()) {
Some(Boundary::Builtin {
func, passed, task, ..
}) => (func + 1 + passed + task.scratch(), *func, *passed),
_ => return Err(VmError::Corrupt),
};
let result = |at: u32| object.stack.get(at as usize).copied().unwrap_or(Value::Nil);
let first = result(slot);
let results = [first, result(slot + 1), result(slot + 2)];
let Some(Boundary::Builtin { task, .. }) = object
.frames
.last_mut()
.and_then(|frame| frame.boundary_mut())
else {
return Err(VmError::Corrupt);
};
if let Task::Collect {
result,
wait,
ended,
left,
} = task
{
if *wait {
*wait = false;
let collector = &self.heap.collector;
*ended = collector.phase == crate::gc::Phase::Pause && !collector.generational;
*left = u32::try_from(self.heap.finalizers.pending.len()).unwrap_or(u32::MAX);
if *left > 0 {
return Ok(Poll::Continue);
}
}
let (result, ended) = (*result, *ended);
let value = match result {
1 => Value::Bool(ended),
4 => Value::Bool(false),
2 => self.new_string(b"incremental".to_vec())?,
3 => self.new_string(b"generational".to_vec())?,
_ => Value::Integer(0),
};
return self.builtin_done(active, &[value]);
}
if let Task::Lib(task) = task {
let saved = std::mem::replace(
&mut **task,
crate::library::LibTask {
work: crate::library::Work::Extreme {
max: false,
best: 0,
next: 0,
},
wait: crate::library::Wait::Nothing,
},
);
return self.finish_lib(
Ctx {
active,
func,
passed,
framed: true,
},
slot,
first,
results[1],
saved,
journal,
);
}
if matches!(task, Task::HostLoad(_)) {
return self.finish_hostload(active, journal);
}
if matches!(task, Task::DoFile) {
let values = object.stack[slot as usize..object.top as usize].to_vec();
return self.builtin_done(active, &values);
}
if let Task::Io(work) = task {
let work = std::mem::replace(work, Box::new(crate::iolib::IoWork::Flush));
return self.finish_io(
Ctx {
active,
func,
passed,
framed: true,
},
slot,
work,
journal,
);
}
let (task, read) = match task {
Task::Load { source } => (Task::Load { source: Vec::new() }, source.len()),
other => (other.clone(), 0),
};
match task {
Task::HostLoad(_) | Task::DoFile | Task::Lib(_) | Task::Io(_) => Err(VmError::Corrupt),
Task::Collect { .. } => Err(VmError::Corrupt),
Task::ToString => match first {
Value::String(_) | Value::Integer(_) | Value::Float(_) => {
let text = self.text_value(first)?;
self.builtin_done(active, &[text])
}
_ => Ok(self.fault(LuaFault::ToString)),
},
Task::Pairs => self.builtin_done(active, &results),
Task::Ipairs { index } => self.builtin_done(active, &ipairs_step(index, first)),
Task::Print { next } => {
if !matches!(
first,
Value::String(_) | Value::Integer(_) | Value::Float(_)
) {
return Ok(self.fault(LuaFault::ToString));
}
let (func, _, _, _) = self.call_site_below(active)?;
let object = self.heap.threads.get_mut(active).ok_or(VmError::Corrupt)?;
object.stack[(func + 1 + next) as usize] = first;
object.stack.truncate(slot as usize);
object.top = slot;
self.print_from(active, next + 1, next, true, journal)
}
Task::Load { .. } => {
let piece = match first {
Value::Nil => None,
value => match text_arg(&self.heap, value) {
Some(piece) if piece.is_empty() => None,
Some(piece) => Some(piece.into_owned()),
None => return Ok(self.fault(LuaFault::Reader)),
},
};
let Some(piece) = piece else {
let source = self.take_source(active)?;
let (func, _, passed, _) = self.call_site_below(active)?;
return match self.load_source(active, func, passed, &source, true)? {
LoadSource::Values(values) => self.builtin_done(active, &values),
};
};
if read == 0 {
let (func, _, passed, _) = self.call_site_below(active)?;
if let Some(message) =
self.mode_failure(active, func, passed, piece.first().copied())?
{
return self.builtin_done(active, &[Value::Nil, message]);
}
}
if read + piece.len() > crate::limits::DEFAULT_SOURCE_BYTES {
let message = self.new_string(
format!(
"source exceeds {} bytes",
crate::limits::DEFAULT_SOURCE_BYTES
)
.into_bytes(),
)?;
return self.builtin_done(active, &[Value::Nil, message]);
}
let charge = piece.len() as u64;
self.make_room(0, charge);
if !self.heap.gc.fits(charge) {
return Ok(self.fault(LuaFault::Memory));
}
self.heap.charge_held(charge);
let mut source = self.take_source(active)?;
source.extend_from_slice(&piece);
self.put_source(active, source)?;
let object = self.heap.threads.get_mut(active).ok_or(VmError::Corrupt)?;
object.stack.truncate(slot as usize);
object.top = slot;
let (func, _, _, _) = self.call_site_below(active)?;
let reader = self
.heap
.threads
.get(active)
.and_then(|thread| thread.stack.get(func as usize + 1))
.copied()
.unwrap_or(Value::Nil);
self.builtin_call(reader, &[], journal)
}
}
}
fn take_source(&mut self, active: Handle<ThreadObj>) -> Result<Vec<u8>, VmError> {
let object = self.heap.threads.get_mut(active).ok_or(VmError::Corrupt)?;
match object
.frames
.last_mut()
.and_then(|frame| frame.boundary_mut())
{
Some(Boundary::Builtin {
task: Task::Load { source },
..
}) => Ok(std::mem::take(source)),
_ => Err(VmError::Corrupt),
}
}
pub(super) fn call_site_below(
&self,
active: Handle<ThreadObj>,
) -> Result<(u32, u8, u32, Value), VmError> {
let frame = self
.heap
.threads
.get(active)
.and_then(|thread| thread.frames.last())
.ok_or(VmError::Corrupt)?;
match frame.boundary() {
Some(Boundary::Builtin { func, passed, .. }) => {
Ok((*func, frame.nresults, *passed, Value::Nil))
}
_ => Err(VmError::Corrupt),
}
}
fn put_source(&mut self, active: Handle<ThreadObj>, source: Vec<u8>) -> Result<(), VmError> {
let object = self.heap.threads.get_mut(active).ok_or(VmError::Corrupt)?;
match object
.frames
.last_mut()
.and_then(|frame| frame.boundary_mut())
{
Some(Boundary::Builtin {
task: Task::Load { source: slot },
..
}) => {
*slot = source;
Ok(())
}
_ => Err(VmError::Corrupt),
}
}
pub(super) fn finish_load_error(
&mut self,
active: Handle<ThreadObj>,
error: Value,
) -> Result<Poll, VmError> {
self.builtin_done(active, &[Value::Nil, error])
}
fn print_from(
&mut self,
active: Handle<ThreadObj>,
mut next: u32,
written: u32,
framed: bool,
journal: &mut Journal,
) -> Result<Poll, VmError> {
let (func, _, passed, _) = if framed {
self.call_site_below(active)?
} else {
self.call_site(active)?
};
let arg = |runtime: &Self, index: u32| {
runtime
.heap
.threads
.get(active)
.and_then(|thread| thread.stack.get((func + 1 + index) as usize))
.copied()
.unwrap_or(Value::Nil)
};
while next < passed {
let value = arg(self, next);
if let Some(method) = index::metamethod(&self.heap, value, b"__tostring") {
self.write_print(func, written, next, false, journal);
let task = Task::Print { next };
if framed {
let object = self.heap.threads.get_mut(active).ok_or(VmError::Corrupt)?;
match object
.frames
.last_mut()
.and_then(|frame| frame.boundary_mut())
{
Some(Boundary::Builtin { task: slot, .. }) => *slot = task,
_ => return Err(VmError::Corrupt),
}
} else {
self.push_builtin(active, task)?;
}
return self.builtin_call(method, &[value], journal);
}
next += 1;
}
self.write_print(func, written, passed, true, journal);
if framed {
self.builtin_done(active, &[])
} else {
self.base_return(active, &[])
}
}
fn write_print(&mut self, func: u32, from: u32, to: u32, end: bool, journal: &mut Journal) {
const GATHER: usize = 64 * 1024;
if from == to && !end {
return;
}
let Some(active) = self.heap.active else {
return;
};
let effect = EffectId {
domain: self.effect_domain,
sequence: self.next_sequence,
};
self.next_sequence = self.next_sequence.saturating_add(1);
let outer = std::mem::replace(&mut self.in_callback, true);
let heap = &self.heap;
let output = &mut self.output;
let _ = journal.commit(effect, i64::from(to - from), || {
let Some(output) = output else {
return 0;
};
let mut gathered = Vec::new();
let stack = heap
.threads
.get(active)
.map_or(&[][..], |thread| thread.stack.as_slice());
for index in from..to {
if index > 0 {
gathered.push(b'\t');
}
let value = stack
.get((func + 1 + index) as usize)
.copied()
.unwrap_or(Value::Nil);
let text = plain_text(heap, value);
if gathered.len() + text.len() > GATHER {
if !gathered.is_empty() {
output(&gathered);
gathered.clear();
}
if text.len() > GATHER {
output(&text);
continue;
}
}
gathered.extend_from_slice(&text);
}
if end {
gathered.push(b'\n');
}
if !gathered.is_empty() {
output(&gathered);
}
0
});
self.in_callback = outer;
}
fn collect_garbage(
&mut self,
active: Handle<ThreadObj>,
option: Value,
) -> Result<Poll, VmError> {
let name: Vec<u8> = match option {
Value::Nil => b"collect".to_vec(),
Value::String(handle) => self.heap.string_bytes(handle).unwrap_or_default().to_vec(),
_ => return self.base_error(active, |r, c| r.bad_type(c, 0, "string")),
};
const OPTIONS: [&[u8]; 10] = [
b"collect",
b"step",
b"count",
b"stop",
b"restart",
b"isrunning",
b"incremental",
b"generational",
b"setpause",
b"setstepmul",
];
let name = name.as_slice();
if OPTIONS.contains(&name) && self.heap.finalizers.running {
return self.base_return(active, &[Value::Nil]);
}
if matches!(name, b"collect" | b"step") {
let (func, _, passed, _) = self.call_site(active)?;
let thread = self.heap.threads.get_mut(active).ok_or(VmError::Corrupt)?;
let live = ((func + 1 + passed) as usize).min(thread.stack.len());
thread.stack[live..].fill(Value::Nil);
}
let (func, _, passed, _) = self.call_site(active)?;
let ctx = Ctx {
active,
func,
passed,
framed: false,
};
let int = |runtime: &Self, index: u32| {
runtime
.opt_int_arg(&ctx, index, 0)
.map(|value| value as i32)
};
let waiting = |result| Task::Collect {
result,
wait: true,
ended: false,
left: 0,
};
let result = match name {
b"collect" => {
crate::gc::request_full(&mut self.heap);
self.push_builtin(active, waiting(0))?;
return Ok(Poll::Continue);
}
b"step" => {
let n = match int(self, 1) {
Ok(n) => n,
Err(next) => return Ok(self.finish_next(active, next)),
};
let gc = &mut self.heap.gc;
gc.sched = if n == 0 {
0
} else {
gc.sched.saturating_add(i64::from(n) * 1024)
};
if n != 0 && gc.sched <= 0 {
Value::Bool(false)
} else {
let generational =
crate::gc::step(&mut self.heap, u64::MAX, self.max_objects, true);
self.push_builtin(active, waiting(if generational { 4 } else { 1 }))?;
return Ok(Poll::Continue);
}
}
b"count" => Value::Float(self.heap.gc.used as f64 / 1024.0),
b"stop" => {
self.heap.gc.auto = false;
Value::Integer(0)
}
b"restart" => {
self.heap.gc.sched = 0;
self.heap.gc.auto = true;
Value::Integer(0)
}
b"isrunning" => Value::Bool(self.heap.gc.auto),
b"incremental" => {
let mut parameters = [0i32; 3];
for (index, parameter) in parameters.iter_mut().enumerate() {
match int(self, 1 + index as u32) {
Ok(value) => *parameter = value,
Err(next) => return Ok(self.finish_next(active, next)),
}
}
let [pause, stepmul, stepsize] = parameters;
let gc = &mut self.heap.gc;
if pause != 0 {
gc.pause = (pause / 4) as u8;
}
if stepmul != 0 {
gc.stepmul = (stepmul / 4) as u8;
}
if stepsize != 0 {
gc.stepsize = stepsize as u8;
}
let previous = gc.generational;
gc.generational = false;
gc.bad = 0;
let collector = &mut self.heap.collector;
if collector.generational && !collector.to_old {
crate::gc::leave_gen(&mut self.heap);
} else {
collector.decide = crate::gc::Decide::None;
}
let name: &[u8] = if previous {
b"generational"
} else {
b"incremental"
};
self.new_string(name.to_vec())?
}
b"setpause" | b"setstepmul" => {
let value = match int(self, 1) {
Ok(value) => value,
Err(next) => return Ok(self.finish_next(active, next)),
};
let gc = &mut self.heap.gc;
let field = if name == b"setpause" {
&mut gc.pause
} else {
&mut gc.stepmul
};
let previous = i64::from(*field) * 4;
*field = (value / 4) as u8;
Value::Integer(previous)
}
b"generational" => {
let mut parameters = [0i32; 2];
for (index, parameter) in parameters.iter_mut().enumerate() {
match int(self, 1 + index as u32) {
Ok(value) => *parameter = value,
Err(next) => return Ok(self.finish_next(active, next)),
}
}
let [minormul, majormul] = parameters;
let gc = &mut self.heap.gc;
if minormul != 0 {
gc.minormul = minormul as u8;
}
if majormul != 0 {
gc.majormul = (majormul / 4) as u8;
}
let previous = gc.generational;
let falling_back = gc.bad > 0;
gc.generational = true;
gc.bad = 0;
if !self.heap.collector.generational && (!previous || falling_back) {
crate::gc::request_full(&mut self.heap);
if self.heap.collector.phase != crate::gc::Phase::Pause {
self.heap.collector.decide = crate::gc::Decide::ToGen;
}
self.push_builtin(active, waiting(if previous { 3 } else { 2 }))?;
return Ok(Poll::Continue);
}
let name: &[u8] = if previous {
b"generational"
} else {
b"incremental"
};
self.new_string(name.to_vec())?
}
other => {
let message = format!("invalid option '{}'", String::from_utf8_lossy(other));
return self.base_error(active, |r, c| r.bad_arg(c, 0, &message));
}
};
self.base_return(active, &[result])
}
fn load_call(
&mut self,
active: Handle<ThreadObj>,
passed: u32,
journal: &mut Journal,
) -> Result<Poll, VmError> {
let (func, _, _, _) = self.call_site(active)?;
let arg = |runtime: &Self, index: u32| -> Value {
if index < passed {
runtime
.heap
.threads
.get(active)
.and_then(|thread| thread.stack.get((func + 1 + index) as usize))
.copied()
.unwrap_or(Value::Nil)
} else {
Value::Nil
}
};
for index in [1, 2] {
let value = arg(self, index);
if !matches!(value, Value::Nil) && text_arg(&self.heap, value).is_none() {
return self.base_error(active, |r, c| r.bad_type(c, index, "string"));
}
}
let chunk = arg(self, 0);
if let Some(source) = text_arg(&self.heap, chunk) {
let source = source.into_owned();
return match self.load_source(active, func, passed, &source, false)? {
LoadSource::Values(values) => self.base_return(active, &values),
};
}
if !chunk.is_function() {
return self.base_error(active, |r, c| r.bad_type(c, 0, "function"));
}
let reader = chunk;
self.push_builtin(active, Task::Load { source: Vec::new() })?;
self.builtin_call(reader, &[], journal)
}
fn mode_failure(
&mut self,
active: Handle<ThreadObj>,
func: u32,
passed: u32,
first: Option<u8>,
) -> Result<Option<Value>, VmError> {
let given = if passed > 2 {
self.heap
.threads
.get(active)
.and_then(|thread| thread.stack.get((func + 3) as usize))
.copied()
.unwrap_or(Value::Nil)
} else {
Value::Nil
};
self.load_mode_failure(given, first)
}
fn load_mode_failure(
&mut self,
given: Value,
first: Option<u8>,
) -> Result<Option<Value>, VmError> {
let mut mode = text_arg(&self.heap, given).map_or_else(|| b"bt".to_vec(), Cow::into_owned);
if let Some(end) = mode.iter().position(|byte| *byte == 0) {
mode.truncate(end);
}
let binary = first == Some(0x1b);
let (kind, letter) = if binary {
("binary", b'b')
} else {
("text", b't')
};
let message = if !mode.contains(&letter) {
let mut message = format!("attempt to load a {kind} chunk (mode is '").into_bytes();
message.extend_from_slice(&mode);
message.extend_from_slice(b"')");
message
} else {
return Ok(None);
};
self.new_string(message).map(Some)
}
fn load_source(
&mut self,
active: Handle<ThreadObj>,
func: u32,
passed: u32,
source: &[u8],
from_reader: bool,
) -> Result<LoadSource, VmError> {
let arg = |runtime: &Self, index: u32| -> Value {
if index < passed {
runtime
.heap
.threads
.get(active)
.and_then(|thread| thread.stack.get((func + 1 + index) as usize))
.copied()
.unwrap_or(Value::Nil)
} else {
Value::Nil
}
};
let text = |runtime: &Self, index: u32| -> Option<Vec<u8>> {
text_arg(&runtime.heap, arg(runtime, index)).map(Cow::into_owned)
};
let name = text(self, 1).map(Cow::Owned).unwrap_or_else(|| {
Cow::Borrowed(if from_reader {
READER_CHUNKNAME
} else {
source
})
});
let env = if passed >= 4 {
arg(self, 3)
} else {
Value::Table(self.heap.globals.ok_or(VmError::Corrupt)?)
};
let chunk_name = match (arg(self, 1), arg(self, 0)) {
(Value::String(handle), _) => ChunkName::Object(handle),
(Value::Nil, Value::String(handle)) if !from_reader => ChunkName::Object(handle),
_ => ChunkName::Bytes(name.to_vec()),
};
self.load_bytes(active, source, &name, arg(self, 2), env, chunk_name)
}
pub(super) fn load_bytes(
&mut self,
active: Handle<ThreadObj>,
source: &[u8],
name: &[u8],
mode: Value,
env: Value,
chunk_name: ChunkName,
) -> Result<LoadSource, VmError> {
if let Some(message) = self.load_mode_failure(mode, source.first().copied())? {
return Ok(LoadSource::Values(vec![Value::Nil, message]));
}
let spec = if source.first() == Some(&0x1b) {
let budget = self
.heap
.gc
.headroom()
.saturating_mul(2)
.saturating_add(source.len() as u64);
match crate::chunk::undump(source, budget) {
Ok(spec) => spec,
Err(refusal) => {
let name = name.to_vec();
let mut message = match name.first() {
Some(b'@' | b'=') => name[1..].to_vec(),
Some(0x1b) => b"binary string".to_vec(),
_ => name,
};
message.extend_from_slice(b": ");
message.extend_from_slice(refusal.as_bytes());
return Ok(LoadSource::Values(vec![
Value::Nil,
self.new_string(message)?,
]));
}
}
} else {
let c_frames = self
.heap
.threads
.get(active)
.map(|thread| {
thread
.frames
.iter()
.filter(|frame| {
matches!(
frame.cold.as_ref().and_then(|cold| cold.boundary.as_ref()),
Some(Boundary::Protect { .. })
)
})
.count()
})
.unwrap_or(0);
let c_frames = u32::try_from(c_frames).unwrap_or(u32::MAX);
match crate::compile::compile_for_load(source, c_frames, self.heap.gc.headroom()) {
Ok(chunk) => chunk.proto,
Err(error) => {
if error.kind == CompileErrorKind::Limit
&& (error.message == "C stack overflow"
|| error.message == crate::limits::COMPILE_MEMORY)
{
if error.message == crate::limits::COMPILE_MEMORY
&& self.terminate_memory_error()
{
return Err(VmError::MemoryLimit);
}
let message = self.new_string(error.message.as_bytes().to_vec())?;
return Ok(LoadSource::Values(vec![Value::Nil, message]));
}
let rendered = error.render_bounded(name, self.heap.max_string);
let message = match rendered {
Some(bytes) => match self.new_string(bytes) {
Ok(value) => value,
Err(VmError::MemoryLimit) => {
self.terminate_memory_error();
self.fault_value(LuaFault::Memory)
}
Err(other) => return Err(other),
},
None => {
self.terminate_memory_error();
self.fault_value(LuaFault::Memory)
}
};
return Ok(LoadSource::Values(vec![Value::Nil, message]));
}
}
};
let chunk_name = if source.first() == Some(&0x1b) {
ChunkName::Unnamed
} else {
chunk_name
};
match self.instantiate(&spec, env, chunk_name) {
Ok(closure) => Ok(LoadSource::Values(vec![Value::Closure(closure)])),
Err(VmError::MemoryLimit) => {
self.terminate_memory_error();
Ok(LoadSource::Values(vec![
Value::Nil,
self.fault_value(LuaFault::Memory),
]))
}
Err(error) => Err(error),
}
}
}
fn ipairs_step(index: i64, value: Value) -> Vec<Value> {
if matches!(value, Value::Nil) {
vec![Value::Nil]
} else {
vec![Value::Integer(index), value]
}
}
#[cfg(test)]
mod tests {
use super::integer_in_base;
use crate::chunkname::chunk_id;
#[test]
fn chunk_ids_match_lua() {
assert_eq!(chunk_id(b"=name"), b"name");
assert_eq!(chunk_id(b"@file.lua"), b"file.lua");
assert_eq!(chunk_id(b"x = "), b"[string \"x = \"]");
assert_eq!(
chunk_id(b"#!shebang\nreturn 1"),
b"[string \"#!shebang...\"]"
);
let long = [b'a'; 80];
assert_eq!(chunk_id(&long).len(), 9 + 45 + 3 + 2);
let mut file = b"@".to_vec();
file.extend_from_slice(&[b'f'; 80]);
assert_eq!(chunk_id(&file).len(), 59);
}
#[test]
fn integers_in_a_base_follow_l_str2int() {
assert_eq!(integer_in_base(b"ff", 16), Some(255));
assert_eq!(integer_in_base(b" -101 ", 2), Some(-5));
assert_eq!(integer_in_base(b"zz", 36), Some(1295));
assert_eq!(integer_in_base(b"ffffffffffffffff", 16), Some(-1));
assert_eq!(integer_in_base(b"10000000000000000", 16), Some(0));
for bad in [&b""[..], b"-", b"1.5", b"0x10", b"2", b"1\0", b"+ 1"] {
assert_eq!(
integer_in_base(bad, if bad == b"2" { 2 } else { 16 }),
None,
"{bad:?}"
);
}
}
}