luau 0.732.0

use luau_common::{BStr, ByteSlice};
use luau_vm::Thread as VmThread;
use luau_vm::debug::LuaDebug;
use luau_vm::state::ProtectedErrorAction;
use luau_vm::{VmErrorResult, VmResult};

use crate::callback::{raise_callback_error, raise_callback_vm_error};
use crate::error::Result;
use crate::lua::runtime::RuntimeData;
use crate::lua::{Lua, LuaRef};
use crate::thread::Thread;

/// Action returned by a debugger callback.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub enum DebugAction {
    /// Continue execution.
    #[default]
    Continue,
    /// Break the current execution.
    Break,
}

/// Information supplied to a step, breakpoint, or debug-interrupt callback.
#[derive(Clone, Copy)]
pub struct DebugContext<'callback> {
    lua: LuaRef<'callback>,
    debug: &'callback LuaDebug,
}

/// Context supplied when a protected call returns an error.
#[derive(Clone, Copy)]
pub struct ProtectedErrorContext<'callback> {
    lua: LuaRef<'callback>,
}

/// A coherent set of Luau debugger callbacks.
pub trait DebugHandler: 'static {
    /// Called after each instruction while single-step mode is enabled.
    fn step(&self, _context: DebugContext<'_>) -> Result<DebugAction> {
        Ok(DebugAction::Continue)
    }

    /// Called when a Luau `BREAK` instruction is encountered.
    fn breakpoint(&self, _context: DebugContext<'_>) -> Result<DebugAction> {
        Ok(DebugAction::Continue)
    }

    /// Called when execution is interrupted by another thread.
    fn interrupt(&self, _context: DebugContext<'_>) -> Result<()> {
        Ok(())
    }

    /// Called when a yieldable protected call returns an error.
    fn protected_error(&self, _context: ProtectedErrorContext<'_>) -> DebugAction {
        DebugAction::Continue
    }
}

type DebugCallback = Box<dyn for<'callback> Fn(DebugContext<'callback>) -> Result<DebugAction>>;
type DebugInterruptCallback = Box<dyn for<'callback> Fn(DebugContext<'callback>) -> Result<()>>;
type ProtectedErrorCallback =
    Box<dyn for<'callback> Fn(ProtectedErrorContext<'callback>) -> DebugAction>;

/// Closure-based builder for a [`DebugHandler`].
#[derive(Default)]
pub struct DebugHooks {
    step: Option<DebugCallback>,
    breakpoint: Option<DebugCallback>,
    interrupt: Option<DebugInterruptCallback>,
    protected_error: Option<ProtectedErrorCallback>,
}

impl DebugHooks {
    /// Creates an empty debugger callback set.
    pub const fn new() -> Self {
        Self {
            step: None,
            breakpoint: None,
            interrupt: None,
            protected_error: None,
        }
    }

    /// Sets the single-step callback.
    #[must_use]
    pub fn on_step<F>(mut self, callback: F) -> Self
    where
        F: for<'callback> Fn(DebugContext<'callback>) -> Result<DebugAction> + 'static,
    {
        self.step = Some(Box::new(callback));
        self
    }

    /// Sets the breakpoint callback.
    #[must_use]
    pub fn on_breakpoint<F>(mut self, callback: F) -> Self
    where
        F: for<'callback> Fn(DebugContext<'callback>) -> Result<DebugAction> + 'static,
    {
        self.breakpoint = Some(Box::new(callback));
        self
    }

    /// Sets the debug-interrupt callback.
    #[must_use]
    pub fn on_interrupt<F>(mut self, callback: F) -> Self
    where
        F: for<'callback> Fn(DebugContext<'callback>) -> Result<()> + 'static,
    {
        self.interrupt = Some(Box::new(callback));
        self
    }

    /// Sets the protected-error callback.
    #[must_use]
    pub fn on_protected_error<F>(mut self, callback: F) -> Self
    where
        F: for<'callback> Fn(ProtectedErrorContext<'callback>) -> DebugAction + 'static,
    {
        self.protected_error = Some(Box::new(callback));
        self
    }
}

impl DebugHandler for DebugHooks {
    fn step(&self, context: DebugContext<'_>) -> Result<DebugAction> {
        self.step
            .as_ref()
            .map(|callback| callback(context))
            .unwrap_or(Ok(DebugAction::Continue))
    }

    fn breakpoint(&self, context: DebugContext<'_>) -> Result<DebugAction> {
        self.breakpoint
            .as_ref()
            .map(|callback| callback(context))
            .unwrap_or(Ok(DebugAction::Continue))
    }

    fn interrupt(&self, context: DebugContext<'_>) -> Result<()> {
        self.interrupt
            .as_ref()
            .map(|callback| callback(context))
            .unwrap_or(Ok(()))
    }

    fn protected_error(&self, context: ProtectedErrorContext<'_>) -> DebugAction {
        self.protected_error
            .as_ref()
            .map(|callback| callback(context))
            .unwrap_or(DebugAction::Continue)
    }
}

impl Lua {
    /// Enables or disables single-step mode on the main thread.
    pub fn set_single_step(&mut self, enabled: bool) {
        self.current_thread().set_single_step(enabled);
    }

    /// Installs the debugger callbacks for this state.
    pub fn set_debug_handler(&mut self, handler: impl DebugHandler) {
        self.runtime
            .callbacks_mut()
            .set_debug_handler(Box::new(handler));
        self.install_callbacks();
    }

    /// Removes the debugger callbacks from this state.
    pub fn remove_debug_handler(&mut self) {
        self.runtime.callbacks_mut().remove_debug_handler();
        self.install_callbacks();
    }
}

impl Thread<'_> {
    /// Enables or disables single-step mode on this thread.
    pub fn set_single_step(&self, enabled: bool) {
        unsafe {
            self.as_vm().single_step(i32::from(enabled));
        }
    }
}

impl<'callback> DebugContext<'callback> {
    pub(in crate::hooks) const fn new(lua: LuaRef<'callback>, debug: &'callback LuaDebug) -> Self {
        Self { lua, debug }
    }

    /// Returns a borrowed reference to the active Luau state.
    pub const fn lua(&self) -> LuaRef<'callback> {
        self.lua
    }

    /// Returns the current source line.
    pub fn current_line(&self) -> Option<usize> {
        usize::try_from(self.debug.currentline).ok()
    }

    /// Returns the current function name, when known.
    pub fn name(&self) -> Option<&'callback BStr> {
        self.debug.name.as_ref().map(|name| name.as_bstr())
    }

    /// Returns the current function's source name.
    pub fn source(&self) -> &'callback BStr {
        self.debug.source.as_bstr()
    }

    /// Returns a shortened form of the source name.
    pub fn short_source(&self) -> &'callback BStr {
        self.debug.short_src().as_bstr()
    }

    /// Returns the current function kind.
    pub fn what(&self) -> &'callback BStr {
        self.debug.what.as_bstr()
    }

    /// Returns the line where the current function was defined.
    pub fn line_defined(&self) -> Option<usize> {
        usize::try_from(self.debug.linedefined).ok()
    }

    /// Returns the number of fixed parameters.
    pub fn parameter_count(&self) -> u8 {
        self.debug.nparams
    }

    /// Returns the number of upvalues.
    pub fn upvalue_count(&self) -> u8 {
        self.debug.nupvals
    }

    /// Returns whether the current function accepts variadic arguments.
    pub fn is_vararg(&self) -> bool {
        self.debug.is_vararg
    }
}

impl<'callback> ProtectedErrorContext<'callback> {
    pub(in crate::hooks) const fn new(lua: LuaRef<'callback>) -> Self {
        Self { lua }
    }

    /// Returns a borrowed reference to the active Luau state.
    pub const fn lua(&self) -> LuaRef<'callback> {
        self.lua
    }
}

pub(in crate::hooks) fn debug_step(thread: &VmThread, debug: &mut LuaDebug) -> VmResult {
    let runtime = RuntimeData::from_thread(thread);
    let Some(handler) = runtime.callbacks().debug_handler() else {
        return Ok(());
    };
    runtime.with_thread(thread, || {
        let context = DebugContext::new(LuaRef::new(thread, runtime), debug);
        match handler.step(context) {
            Ok(action) => apply_debug_action(thread, action),
            Err(error) => raise_callback_error(thread, runtime, error),
        }
    })
}

pub(in crate::hooks) fn debug_break(thread: &VmThread, debug: &mut LuaDebug) -> VmResult {
    let runtime = RuntimeData::from_thread(thread);
    let Some(handler) = runtime.callbacks().debug_handler() else {
        return Ok(());
    };
    runtime.with_thread(thread, || {
        let context = DebugContext::new(LuaRef::new(thread, runtime), debug);
        match handler.breakpoint(context) {
            Ok(action) => apply_debug_action(thread, action),
            Err(error) => raise_callback_error(thread, runtime, error),
        }
    })
}

pub(in crate::hooks) fn debug_interrupt(thread: &VmThread, debug: &mut LuaDebug) -> VmErrorResult {
    let runtime = RuntimeData::from_thread(thread);
    let Some(handler) = runtime.callbacks().debug_handler() else {
        return Ok(());
    };
    runtime.with_thread(thread, || {
        let context = DebugContext::new(LuaRef::new(thread, runtime), debug);
        handler
            .interrupt(context)
            .or_else(|error| raise_callback_vm_error(thread, runtime, error))
    })
}

pub(in crate::hooks) fn debug_protected_error(thread: &VmThread) -> ProtectedErrorAction {
    let runtime = RuntimeData::from_thread(thread);
    runtime.with_thread(thread, || {
        let action = runtime
            .callbacks()
            .debug_handler()
            .map(|handler| {
                handler.protected_error(ProtectedErrorContext::new(LuaRef::new(thread, runtime)))
            })
            .unwrap_or(DebugAction::Continue);
        match action {
            DebugAction::Continue => ProtectedErrorAction::Continue,
            DebugAction::Break => ProtectedErrorAction::Break,
        }
    })
}

fn apply_debug_action(thread: &VmThread, action: DebugAction) -> VmResult {
    match action {
        DebugAction::Continue => Ok(()),
        DebugAction::Break => unsafe { thread.break_current().map(|_| ()) },
    }
}