go-lib 0.6.1

// SPDX-License-Identifier: Apache-2.0
//! x86-64 context switch primitives — ported from `runtime/asm_amd64.s` and
//! `runtime/preempt_amd64.s`.
//!
//! Public entry points:
//! - [`gogo`]                     — restore a saved `Gobuf` and resume a goroutine.
//! - [`mcall`]                    — save current G, switch to g0's stack, call a fn.
//! - [`async_preempt_trampoline`] — save all GPRs + XMMs, call `async_preempt2`,
//!   restore, ret to interrupted PC.  *(v0.2.0 — Step 4)*
//! - [`systemstack`]              — run a closure on g0's stack.
//!
//! ## Design vs Go's approach
//!
//! Go uses the `FS` segment register (Linux) or `GS` (macOS) as a TLS pointer
//! to the current G, accessed from assembly via `get_tls`.  We use a Rust
//! `thread_local!` (`CURRENT_G` in `g.rs`) updated from the Rust wrapper,
//! keeping the naked asm free of OS-specific TLS segment tricks.
//!
//! ## Calling convention
//!
//! **System V AMD64 (Linux, macOS)** — arguments in `rdi`, `rsi`, `rdx`, `rcx`, `r8`, `r9`.
//! Caller-saved: `rax`, `rcx`, `rdx`, `rsi`, `rdi`, `r8`–`r11`.
//! Callee-saved: `rbx`, `rbp`, `r12`–`r15`.
//! Stack: 16-byte aligned before a `call`.  No shadow space.
//!
//! **Microsoft x64 (Windows)** — arguments in `rcx`, `rdx`, `r8`, `r9`.
//! Caller-saved: `rax`, `rcx`, `rdx`, `r8`–`r11`.
//! Callee-saved: `rbx`, `rbp`, `rdi`, `rsi`, `r12`–`r15`, `xmm6`–`xmm15`.
//! Stack: 16-byte aligned before a `call`.  **Caller must allocate 32 bytes
//! of shadow space below RSP before any `call`** — the callee may write its
//! first four register arguments there.  Without it a callee that spills its
//! first argument (`rcx`) would write to `[rsp+8]`, which equals `g0.stack.hi+8`
//! — just past the end of the VirtualAlloc region → `STATUS_ACCESS_VIOLATION`.
//!
//! ## Gobuf field offsets (verified by compile-time assertions in `g.rs`)
//! ```text
//!  0  sp
//!  8  pc
//! 16  g
//! 24  ctxt
//! 32  ret
//! 40  lr  (unused on x86-64)
//! 48  bp
//! ```
//!
//! ## Assembly syntax
//! Rust's `naked_asm!` on x86-64 uses Intel syntax by default.

use std::ptr::addr_of_mut;

use super::g::{
    set_current_g, Gobuf, G,
    G0_SCHED,
    GOBUF_BP_OFFSET, GOBUF_G_OFFSET,
    GOBUF_PC_OFFSET, GOBUF_REGS_OFFSET, GOBUF_SP_OFFSET,
};
#[cfg(windows)]
use super::g::{G_STACK_HI_OFFSET, G_STACK_LO_OFFSET};

// ---------------------------------------------------------------------------
// gogo — restore saved state and jump
// ---------------------------------------------------------------------------

/// Restore register state from `buf` and resume execution at `buf.pc`.
///
/// Ported from `runtime·gogo` in `runtime/asm_amd64.s`.
///
/// Register usage (System V AMD64 — Linux / macOS):
/// - `rdi` = buf (*mut Gobuf, first arg)
///
/// Register usage (Microsoft x64 — Windows):
/// - `rcx` = buf (*mut Gobuf, first arg)
///
/// Common: `rax` = scratch (target pc), `rbp` / `rsp` restored from Gobuf.
///
/// ## Callee-saved register restoration
///
/// `gogo_asm` resumes execution at `buf.pc`, which (for a `mcall`-yielded
/// goroutine) is the instruction *immediately after* `call mcall_asm`.  The
/// Rust function that called `mcall` follows the platform ABI, which means
/// it may hold live values in callee-saved registers across the call.  We
/// restore those slots here so the caller's frame sees the exact register
/// state it left behind, not whatever the scheduler happened to leave there.
///
/// System V AMD64 (Linux/macOS) callee-saved GPRs: RBX, R12, R13, R14, R15
/// (plus RBP, which we already restore from `bp`).  No callee-saved XMM/YMM.
// System V AMD64 ABI (Linux, macOS): first argument in rdi.
#[cfg(not(windows))]
#[unsafe(naked)]
pub(crate) unsafe extern "C" fn gogo_asm(buf: *mut Gobuf) -> ! {
    core::arch::naked_asm!(
        // Load callee-saved GPRs from gobuf.regs[..] BEFORE switching stacks.
        // Order matches mcall_asm's save order: [rbx, r12, r13, r14, r15].
        "mov rbx, [rdi + {regs} + 0]",
        "mov r12, [rdi + {regs} + 8]",
        "mov r13, [rdi + {regs} + 16]",
        "mov r14, [rdi + {regs} + 24]",
        "mov r15, [rdi + {regs} + 32]",
        "mov rax, [rdi + {pc}]",   // rax = gobuf.pc  (load before stack switch)
        "mov rbp, [rdi + {bp}]",   // rbp = gobuf.bp  (frame pointer)
        "mov rsp, [rdi + {sp}]",   // rsp = gobuf.sp  (stack switch — do last)
        "jmp rax",                  // jump to pc — never returns
        pc   = const GOBUF_PC_OFFSET,
        bp   = const GOBUF_BP_OFFSET,
        sp   = const GOBUF_SP_OFFSET,
        regs = const GOBUF_REGS_OFFSET,
    )
}

// Microsoft x64 ABI (Windows): first argument in rcx.
// Microsoft x64 callee-saved GPRs add RDI and RSI vs. System V, plus the
// full 128 bits of XMM6–XMM15.
#[cfg(windows)]
#[unsafe(naked)]
pub(crate) unsafe extern "C" fn gogo_asm(buf: *mut Gobuf) -> ! {
    core::arch::naked_asm!(
        // Restore callee-saved GPRs: [rbx, rdi, rsi, r12, r13, r14, r15].
        "mov rbx, [rcx + {regs} + 0]",
        "mov rdi, [rcx + {regs} + 8]",
        "mov rsi, [rcx + {regs} + 16]",
        "mov r12, [rcx + {regs} + 24]",
        "mov r13, [rcx + {regs} + 32]",
        "mov r14, [rcx + {regs} + 40]",
        "mov r15, [rcx + {regs} + 48]",
        // Restore callee-saved XMM6–15 (full 128 bits, saved by mcall_asm).
        "movdqu xmm6,  [rcx + {regs} + 56]",
        "movdqu xmm7,  [rcx + {regs} + 72]",
        "movdqu xmm8,  [rcx + {regs} + 88]",
        "movdqu xmm9,  [rcx + {regs} + 104]",
        "movdqu xmm10, [rcx + {regs} + 120]",
        "movdqu xmm11, [rcx + {regs} + 136]",
        "movdqu xmm12, [rcx + {regs} + 152]",
        "movdqu xmm13, [rcx + {regs} + 168]",
        "movdqu xmm14, [rcx + {regs} + 184]",
        "movdqu xmm15, [rcx + {regs} + 200]",
        "mov rax, [rcx + {pc}]",   // rax = gobuf.pc  (load before stack switch)
        "mov rbp, [rcx + {bp}]",   // rbp = gobuf.bp  (frame pointer)
        "mov rsp, [rcx + {sp}]",   // rsp = gobuf.sp  (stack switch — do last)
        "jmp rax",                  // jump to pc — never returns
        pc   = const GOBUF_PC_OFFSET,
        bp   = const GOBUF_BP_OFFSET,
        sp   = const GOBUF_SP_OFFSET,
        regs = const GOBUF_REGS_OFFSET,
    )
}

// ---------------------------------------------------------------------------
// mcall — save current G's state and switch to g0
// ---------------------------------------------------------------------------

/// Save the current goroutine's registers into `g_sched`, switch to g0's
/// stack, and call `fn_ptr(g)`.  Never returns via the normal path.
///
/// The return type is `()` (not `!`) deliberately: the Rust compiler must
/// generate a proper epilogue for `mcall()` *after* the `call mcall_asm`
/// instruction.  When `gogo` later resumes a goroutine it jumps to
/// `g_sched.pc`, which points at that epilogue.  Executing the epilogue
/// unwinds the `mcall` and caller (`gosched`/`gopark`) frames normally —
/// exactly the same sequence Go uses.
///
/// Ported from `runtime·mcall` in `runtime/asm_amd64.s`.
///
/// ## Calling conventions
///
/// **System V AMD64 (Linux, macOS)** — argument registers on entry:
/// - `rdi` = g, `rsi` = g_sched, `rdx` = g0_gobuf, `rcx` = fn_ptr
///
/// **Microsoft x64 (Windows)** — argument registers on entry:
/// - `rcx` = g, `rdx` = g_sched, `r8` = g0_gobuf, `r9` = fn_ptr
///
/// In both ABIs `[rsp]` on entry holds the return address pushed by the
/// `call mcall_asm` instruction.  Caller SP = `rsp + 8`.
// System V AMD64 ABI (Linux, macOS): args in rdi, rsi, rdx, rcx.
//
// ## Callee-saved register save
//
// `mcall_asm` is invoked by a Rust function that obeys the platform ABI: it
// expects callee-saved GPRs (RBX, R12–R15 on System V, plus RBP) to be
// preserved across the call.  But the goroutine is then yielded — the
// scheduler will run arbitrary code on this M and may clobber every register.
// Without saving the callee-saves here, the caller would resume with
// scheduler garbage in RBX/R12–R15 → corruption.
//
// We save them into `g_sched.regs[..]` (slots [0..5]).  `gogo_asm` restores
// them when the goroutine is resumed.  Order: [rbx, r12, r13, r14, r15].
#[cfg(not(windows))]
#[unsafe(naked)]
pub(crate) unsafe extern "C" fn mcall_asm(
    _g:        *mut G,
    _g_sched:  *mut Gobuf,
    _g0_gobuf: *mut Gobuf,
    _fn_ptr:   unsafe extern "C" fn(*mut G),
) {
    core::arch::naked_asm!(
        // ── save current goroutine's context into g_sched (rsi) ──────────
        "mov rax,          [rsp]",         // rax = return address (caller PC)
        "mov [rsi + {pc}], rax",           // g_sched.pc = return address
        "lea rax,          [rsp + 8]",     // rax = caller SP (before call pushed ret addr)
        "mov [rsi + {sp}], rax",           // g_sched.sp = caller SP
        "mov [rsi + {bp}], rbp",           // g_sched.bp = frame pointer
        "mov [rsi + {g}],  rdi",           // g_sched.g  = g

        // ── save callee-saved GPRs (System V AMD64): rbx, r12-r15 ────────
        "mov [rsi + {regs} + 0],  rbx",
        "mov [rsi + {regs} + 8],  r12",
        "mov [rsi + {regs} + 16], r13",
        "mov [rsi + {regs} + 24], r14",
        "mov [rsi + {regs} + 32], r15",

        // ── switch to g0's stack (rdx = g0_gobuf) ────────────────────────
        "mov rsp, [rdx + {sp}]",           // rsp = g0.sp (stack switch)
        "mov rbp, [rdx + {bp}]",           // rbp = g0.bp

        // ── call fn_ptr(g) on g0's stack ─────────────────────────────────
        // rdi = g (first argument, System V: first arg in rdi ✓)
        // rcx = fn_ptr
        "call rcx",
        // If fn_ptr returns (shutdown path: schedule() returned), exit this
        // OS thread cleanly rather than hitting an illegal instruction.
        "call {m_exit}",

        pc     = const GOBUF_PC_OFFSET,
        sp     = const GOBUF_SP_OFFSET,
        bp     = const GOBUF_BP_OFFSET,
        g      = const GOBUF_G_OFFSET,
        regs   = const GOBUF_REGS_OFFSET,
        m_exit = sym crate::runtime::sched::m_thread_exit,
    )
}

// Microsoft x64 ABI (Windows): args in rcx, rdx, r8, r9.
//
// Microsoft x64 callee-saved registers: RBX, RBP, RDI, RSI, R12–R15, plus
// the full 128 bits of XMM6–XMM15.  We save all of them except RBP (already
// saved separately in `g_sched.bp`); the XMM slots live in the same
// `gobuf.regs` array after the GPRs and are accessed with `movdqu` because
// the array is only 8-byte aligned.  Missing the XMM saves manifested as
// STATUS_HEAP_CORRUPTION on Windows CI: a resumed goroutine continued with
// scheduler garbage in XMM6+, corrupting whatever its vectorised code
// touched next.
#[cfg(windows)]
#[unsafe(naked)]
pub(crate) unsafe extern "C" fn mcall_asm(
    _g:        *mut G,
    _g_sched:  *mut Gobuf,
    _g0_gobuf: *mut Gobuf,
    _fn_ptr:   unsafe extern "C" fn(*mut G),
) {
    core::arch::naked_asm!(
        // ── save current goroutine's context into g_sched (rdx) ──────────
        // rcx = g, rdx = g_sched, r8 = g0_gobuf, r9 = fn_ptr
        "mov rax,          [rsp]",         // rax = return address (caller PC)
        "mov [rdx + {pc}], rax",           // g_sched.pc = return address
        "lea rax,          [rsp + 8]",     // rax = caller SP
        "mov [rdx + {sp}], rax",           // g_sched.sp = caller SP
        "mov [rdx + {bp}], rbp",           // g_sched.bp = frame pointer
        "mov [rdx + {g}],  rcx",           // g_sched.g  = g

        // ── save callee-saved GPRs (Microsoft x64): rbx, rdi, rsi, r12-r15
        "mov [rdx + {regs} + 0],  rbx",
        "mov [rdx + {regs} + 8],  rdi",
        "mov [rdx + {regs} + 16], rsi",
        "mov [rdx + {regs} + 24], r12",
        "mov [rdx + {regs} + 32], r13",
        "mov [rdx + {regs} + 40], r14",
        "mov [rdx + {regs} + 48], r15",

        // ── save callee-saved XMM6–15 (Microsoft x64, full 128 bits each) ─
        // movdqu: the regs array is only 8-byte aligned.
        "movdqu [rdx + {regs} + 56],  xmm6",
        "movdqu [rdx + {regs} + 72],  xmm7",
        "movdqu [rdx + {regs} + 88],  xmm8",
        "movdqu [rdx + {regs} + 104], xmm9",
        "movdqu [rdx + {regs} + 120], xmm10",
        "movdqu [rdx + {regs} + 136], xmm11",
        "movdqu [rdx + {regs} + 152], xmm12",
        "movdqu [rdx + {regs} + 168], xmm13",
        "movdqu [rdx + {regs} + 184], xmm14",
        "movdqu [rdx + {regs} + 200], xmm15",

        // ── switch to g0's stack (r8 = g0_gobuf) ─────────────────────────
        "mov rsp, [r8 + {sp}]",            // rsp = g0.sp (= g0.stack.hi — top of allocation)
        "mov rbp, [r8 + {bp}]",            // rbp = g0.bp

        // ── restore TEB stack bounds to g0's stack ────────────────────────
        // `gogo()` sets TEB to the goroutine's stack bounds before every
        // context switch.  Now that we're on g0's stack, we must update the
        // TEB to g0's bounds before any code runs on g0.  Without this,
        // Windows sees RSP outside [StackLimit, StackBase) and either raises
        // a spurious stack-overflow or attempts to auto-grow into unmapped
        // memory (STATUS_ACCESS_VIOLATION, fault-type write).
        //
        // G.stack.lo / G.stack.hi are at byte offsets G_STACK_LO_OFFSET (0)
        // and G_STACK_HI_OFFSET (8) because G is #[repr(C)] with `stack:
        // Stack` as its first field.  g0_gobuf.g (GOBUF_G_OFFSET = 16) is
        // the back-pointer to g0's G struct, set by G::new().
        //
        // r10 / r11 are caller-saved on Windows x64 — safe to clobber here.
        "mov r10, [r8 + {g}]",             // r10 = G0* (g0_gobuf.g)
        "mov r11, [r10 + {stack_lo}]",     // r11 = g0.stack.lo (new StackLimit)
        "mov r10, [r10 + {stack_hi}]",     // r10 = g0.stack.hi (new StackBase)
        "mov qword ptr gs:[0x10], r11",    // TEB.StackLimit = g0.stack.lo
        "mov qword ptr gs:[0x08], r10",    // TEB.StackBase  = g0.stack.hi

        // ── call fn_ptr(g) on g0's stack ─────────────────────────────────
        // Microsoft x64 ABI requires the *caller* to allocate 32 bytes of
        // "home space" (shadow space) before any CALL.  Without it the callee
        // prologue writes rcx → [rsp+8], which equals g0.stack.hi+8 — one
        // byte past the VirtualAlloc region → STATUS_ACCESS_VIOLATION.
        "sub rsp, 32",                     // allocate shadow space (keeps rsp 16-byte aligned)
        // rcx = g (still holds g — Microsoft x64 first arg in rcx ✓)
        // r9  = fn_ptr
        "call r9",
        // If fn_ptr returns (shutdown path: schedule() returned), exit this
        // OS thread cleanly rather than hitting an illegal instruction.
        "call {m_exit}",

        pc       = const GOBUF_PC_OFFSET,
        sp       = const GOBUF_SP_OFFSET,
        bp       = const GOBUF_BP_OFFSET,
        g        = const GOBUF_G_OFFSET,
        regs     = const GOBUF_REGS_OFFSET,
        stack_lo = const G_STACK_LO_OFFSET,
        stack_hi = const G_STACK_HI_OFFSET,
        m_exit   = sym crate::runtime::sched::m_thread_exit,
    )
}

// ---------------------------------------------------------------------------
// Public wrappers
// ---------------------------------------------------------------------------

/// Resume goroutine `g` by restoring its saved register state and jumping.
///
/// Updates `CURRENT_G` before the context switch so any code running after
/// the switch sees the correct current goroutine.  The caller must have
/// initialised `g.sched.sp` and `g.sched.pc` before calling.
///
/// On Windows, the TEB `StackBase` / `StackLimit` fields are updated to
/// reflect the goroutine's custom stack bounds before the RSP switch.
/// Windows' exception dispatcher (`RtlDispatchException`) validates that
/// the faulting RSP is inside `[TEB.StackLimit, TEB.StackBase)` before
/// walking frame-based handlers.  Without this update, any `catch_unwind`
/// inside a goroutine is silently bypassed and the process terminates with
/// `0xe06d7363` (STATUS_CPP_EH_EXCEPTION).
///
/// Ported from the `execute` → `gogo` path in `runtime/proc.go` +
/// `runtime/asm_amd64.s`.
pub(crate) unsafe fn gogo(g: *mut G) -> ! {
    unsafe {
        set_current_g(g);

        // Windows only: tell the OS about the goroutine's stack region so
        // that SEH can find exception handlers while the goroutine runs.
        #[cfg(windows)]
        {
            // Read the fields directly — Stack doesn't implement Copy, and
            // moving out of a raw-pointer dereference requires Copy or
            // ptr::read.  The two usize fields are trivially readable.
            let stack_lo = (*g).stack.lo;
            let stack_hi = (*g).stack.hi;
            // GS:[0x08] = StackBase (exclusive high address of the stack).
            // GS:[0x10] = StackLimit (current lowest committed stack address).
            std::arch::asm!(
                "mov qword ptr gs:[0x08], {hi}",
                "mov qword ptr gs:[0x10], {lo}",
                lo = in(reg) stack_lo,
                hi = in(reg) stack_hi,
                options(nostack, preserves_flags),
            );
        }

        gogo_asm(addr_of_mut!((*g).sched))
    }
}

/// Save the current goroutine's state into `g.sched` and switch to g0's
/// stack, calling `fn_ptr(g)` there.
///
/// `fn_ptr` must eventually call `schedule()` or hand off via `gogo()` and
/// must not return to its caller.
///
/// The return type is `()` (not `!`) for the same reason as `mcall_asm`: the
/// compiler must emit an epilogue (`leave; ret`) after `callq mcall_asm` so
/// that `gogo` can resume the goroutine by jumping to that epilogue and
/// returning through the call stack normally.
///
/// Requires `G0_SCHED` to be initialised by `M::new` (step 6); panics in
/// debug builds if it has not been set yet.
///
/// Ported from `runtime·mcall` in `runtime/proc.go` + `runtime/asm_amd64.s`.
///
/// ## Why `#[inline(never)]` is load-bearing
///
/// `mcall` reads the thread-local `G0_SCHED` and passes the resulting gobuf
/// pointer to `mcall_asm`, which switches RSP onto that g0 stack.  The call
/// to `mcall_asm` is a *suspension point*: the goroutine may resume on a
/// different OS thread (another M `gogo`s it).  If `mcall` is inlined into a
/// caller that yields more than once (e.g. a `gosched()` loop), LLVM CSEs the
/// `G0_SCHED` TLS accessor and keeps the **slot address** in a callee-saved
/// register across the suspension.  After a cross-thread resume that cached
/// address still points at the *old* thread's slot, so the next yield runs
/// the scheduler on the old M's g0 stack — corrupting the live scheduler
/// frames of whatever that M is doing (observed as SIGBUS `ret`-to-heap in
/// release builds with GOMAXPROCS ≥ 2).  Keeping `mcall` out-of-line forces
/// the TLS slot address to be re-derived on the current thread at every
/// suspension, the same rule Go enforces by forbidding TLS caching across
/// `mcall`/`gopark` in its compiler.
#[inline(never)]
pub(crate) unsafe fn mcall(g: *mut G, fn_ptr: unsafe extern "C" fn(*mut G)) {
    unsafe {
        let g_sched  = addr_of_mut!((*g).sched);
        let g0_gobuf = G0_SCHED.with(|c| c.get());
        debug_assert!(
            !g0_gobuf.is_null(),
            "mcall: G0_SCHED is null — M::new must be called before spawning goroutines (step 6)",
        );
        mcall_asm(g, g_sched, g0_gobuf, fn_ptr);
        // mcall_asm switches to g0 and calls fn_ptr (which calls schedule,
        // an infinite loop).  Execution never reaches here during normal
        // forward flow.  When gogo() later resumes this goroutine it jumps
        // directly to the `leave; ret` epilogue of this function (the
        // instruction after `callq mcall_asm`), unwinding the frame chain
        // back to the goroutine's user code.
    }
}

// ---------------------------------------------------------------------------
// systemstack — run a closure on g0's stack
// ---------------------------------------------------------------------------

/// Low-level stack switch: save goroutine RSP/RBP, switch to `g0_sp`, call
/// `thunk(arg)` on g0's stack, then restore the goroutine's RSP/RBP and return.
///
/// ## Register layout on entry (System V AMD64 — Linux / macOS)
/// - `rdi` = g0_sp   (target stack pointer, aligned to 16 bytes inside)
/// - `rsi` = arg     (opaque closure pointer, forwarded to thunk)
/// - `rdx` = thunk   (function to call on g0's stack)
///
/// ## Register layout on entry (Microsoft x64 — Windows)
/// - `rcx` = g0_sp
/// - `rdx` = arg
/// - `r8`  = thunk
///
/// ## Safety
/// `g0_sp` must be a valid, accessible stack address for this OS thread's g0.
/// `thunk` must not panic or longjmp.
///
/// Ported from `runtime·systemstack` in `runtime/asm_amd64.s`.
#[cfg(not(windows))]
#[allow(dead_code)] // called by systemstack; no callers until systemstack is used
#[unsafe(naked)]
unsafe extern "C" fn systemstack_call(
    _g0_sp: usize,
    _arg:   *mut (),
    _thunk: unsafe extern "C" fn(*mut ()),
) {
    core::arch::naked_asm!(
        // Save goroutine frame pointer on goroutine's stack, then record the
        // goroutine's current RSP in RBP for restoration after the call.
        "push rbp",           // [goroutine stack] save old RBP
        "mov  rbp, rsp",      // RBP = goroutine SP (post-push)
        // Switch to g0 stack (rdi = g0_sp, already the arg).
        "and  rdi, -16",      // 16-byte-align the target SP
        "mov  rsp, rdi",      // RSP now on g0's stack
        "sub  rsp, 8",        // maintain 16-byte alignment before CALL
        // Call thunk(arg): arg is in rsi, thunk is in rdx.
        "mov  rdi, rsi",      // 1st arg: rdi = arg
        "call rdx",           // call thunk(arg) — ret addr lands on g0 stack
        // Restore goroutine stack.
        "mov  rsp, rbp",      // RSP = saved goroutine SP
        "pop  rbp",           // restore old RBP
        "ret",
    )
}

/// Windows x64 variant: rcx=g0_sp, rdx=arg, r8=thunk.
#[cfg(windows)]
#[allow(dead_code)] // called by systemstack; no callers until systemstack is used
#[unsafe(naked)]
unsafe extern "C" fn systemstack_call(
    _g0_sp: usize,
    _arg:   *mut (),
    _thunk: unsafe extern "C" fn(*mut ()),
) {
    core::arch::naked_asm!(
        "push rbp",
        "mov  rbp, rsp",
        "and  rcx, -16",
        "mov  rsp, rcx",
        // 32-byte shadow space + 8-byte alignment pad for CALL.
        "sub  rsp, 40",
        "mov  rcx, rdx",      // 1st arg: rcx = arg
        "call r8",            // call thunk(arg)
        "mov  rsp, rbp",
        "pop  rbp",
        "ret",
    )
}

/// Run `f` on the M's g0 (system) stack, then return to the current goroutine.
///
/// If already on g0 (scheduler context — `CURRENT_G` is null), `f` is called
/// directly without any stack switch.
///
/// ## How the switch works
///
/// `gogo` saves g0's stack pointer into the thread-local `G0_SCHED.sp` every
/// time it switches into a goroutine.  While the goroutine runs, g0 is idle —
/// its stack memory is allocated and valid, just not active.  `systemstack`
/// reads that saved SP, uses `systemstack_call` (a naked helper) to swap RSP,
/// calls `f` on g0's stack, and restores RSP before returning.
///
/// The closure `f` is stored in a `ManuallyDrop` slot on the goroutine's own
/// stack.  The goroutine stack memory remains valid throughout the switch (only
/// RSP changes), so the pointer passed to the thunk is always live.
///
/// Ported from `systemstack` in `runtime/asm_amd64.s`.
#[allow(dead_code)] // future callers: stack growth, signal handlers, GC hooks
pub(crate) unsafe fn systemstack<F: FnOnce()>(f: F) {
    // Already on g0 — call directly without switching stacks.
    if super::g::current_g().is_null() {
        f();
        return;
    }

    let g0_sp = unsafe { (*super::g::g0_sched()).sp };
    debug_assert!(g0_sp != 0, "systemstack: g0_sched.sp is 0 — M not initialised");

    // Keep `f` on the goroutine's stack; it stays accessible after the RSP
    // switch because the goroutine stack is allocated memory, not the
    // currently-active stack in any CPU sense.
    let mut slot = std::mem::ManuallyDrop::new(f);
    let arg = std::ptr::addr_of_mut!(slot) as *mut ();

    /// Thunk called on g0's stack: reads the closure out of `arg` and calls it.
    ///
    /// SAFETY: `arg` points to a `ManuallyDrop<F>` on the goroutine's stack,
    /// which is still valid memory even though RSP has been switched to g0.
    unsafe extern "C" fn thunk<F: FnOnce()>(arg: *mut ()) {
        let f = unsafe { std::ptr::read(arg as *mut F) };
        f();
    }

    unsafe { systemstack_call(g0_sp, arg, thunk::<F>) };
}

// ---------------------------------------------------------------------------
// async_preempt_trampoline — Step 4: async signal-based preemption
// ---------------------------------------------------------------------------

/// Trampoline injected by the SIGURG handler to preempt a running goroutine.
///
/// This is the Unix (System V) body.  Windows has no POSIX signals, so there
/// the equivalent trampoline is injected via `SuspendThread` + `SetThreadContext`
/// (`preempt_m_windows` in `sched.rs`) and the body differs only in its call-site
/// shadow space — see the `#[cfg(windows)]` sibling at the end of this file.
#[cfg(not(windows))]
///
/// The SIGURG handler redirects the goroutine's `RIP` to this function and
/// pushes the original `RIP` onto the goroutine's stack (decrements `RSP` and
/// writes the original PC to `[RSP]`).  When the goroutine resumes after the
/// signal returns, execution begins here — exactly as if the goroutine had been
/// called with a normal `call` instruction.
///
/// ## Register layout on entry
/// - `[RSP]`   = original `RIP` (the preemption point; serves as the return address)
/// - `RSP+8 .. RSP+136` = the interrupted function's System V red zone,
///   preserved untouched (redirect_to_async_preempt stores the resume PC at
///   `rsp − 136`, not `rsp − 8`, so this frame cannot clobber red-zone
///   locals of an interrupted leaf function; the final `ret 128` undoes the
///   displacement)
/// - `RSP+136..` = goroutine's live stack at the moment of preemption
/// - All other registers: unchanged (intact from the interrupted state)
///
/// ## Frame layout (built by this function)
/// ```text
/// [RSP+0  .. RSP+7]:    RFLAGS (8 B, saved by pushfq)
/// [RSP+8  .. RSP+127]:  15 × 8 B GPRs: RBP R15 R14 R13 R12 R11 R10 R9
///                                        R8  RDI RSI RDX RCX RBX RAX
/// [RSP+128 .. RSP+383]: 264 B XMM area (16 × 16 B data + 8 B pad)
/// ```
///
/// Total frame: 8 + 120 + 264 = 392 B.
///
/// ## Stack alignment
/// On entry `RSP % 16 == 8` (redirect pushed original RIP).
/// After `pushfq` (8 B): `RSP % 16 == 0`.
/// After 15 GPR pushes (120 B): `RSP % 16 == 8`.
/// After `sub rsp, 264` (264 B): `RSP % 16 == 0`.  Correct for a call site.
///
/// Ported from the auto-generated `asyncPreempt` in `runtime/preempt_amd64.s`.
#[unsafe(naked)]
pub(crate) unsafe extern "C" fn async_preempt_trampoline() {
    // NOTE: this attribute / function body is only compiled on non-Windows
    // (gated by the #[cfg(not(windows))] on the doc-comment block above).
    core::arch::naked_asm!(
        // ── save RFLAGS (must be first — before any flag-modifying instruction)
        //
        // The goroutine was interrupted at an arbitrary instruction.  Many
        // Rust constructs read condition-code flags that were set by a
        // preceding arithmetic instruction.  For example, the standard
        // RangeInclusive::next() (spec_next) calls Step::forward_unchecked,
        // which does:
        //
        //   addl %ecx, %edi      ← sets OF, SF, ZF, CF
        //   movl %edi, -N(%rbp)  ← no flags effect
        //   seto %al             ← reads OF (overflow check)
        //   cmpl $0, %ecx
        //   setl %cl             ← reads SF/OF
        //
        // If async preemption fires between the `addl` and `seto`, and we
        // do not save RFLAGS here, the scheduler's Rust code (which freely
        // uses arithmetic) clobbers OF.  On resume, `seto` reads the wrong
        // OF → wrong `al` → wrong branch → wrong return value.  Observed:
        // the iterator's `next()` returns None early (wrong sum) or jumps
        // to an `unreachable_unchecked` abort path (SIGABRT).
        //
        // Entry invariant: RSP%16 == 8 (redirect_to_async_preempt pushed the
        // original RIP).  pushfq stores full 64-bit RFLAGS and decrements RSP
        // by 8, making RSP%16 == 0.
        "pushfq",
        // RSP % 16 == 0

        // ── save all general-purpose registers (15 pushes = 120 B) ──────────
        "push rbp",
        "push r15",
        "push r14",
        "push r13",
        "push r12",
        "push r11",
        "push r10",
        "push r9",
        "push r8",
        "push rdi",
        "push rsi",
        "push rdx",
        "push rcx",
        "push rbx",
        "push rax",
        // After pushfq + 15 GPR pushes = 128 B total.  RSP%16 = 8 − 128 mod 16
        // = 8 − 0 = 8.  Need RSP%16 == 0 before the `call`.  Allocating 264 B
        // (= 256 for XMM data + 8 B alignment pad) gives RSP%16 = 8 − 264%16
        // = 8 − 8 = 0.  ✓

        // ── allocate XMM save area (264 B: 256 data + 8 B alignment pad) ────
        "sub rsp, 264",
        // RSP % 16 == 0 — correct call-site alignment

        // ── save XMM registers ───────────────────────────────────────────────
        "movdqu [rsp + 0],   xmm0",
        "movdqu [rsp + 16],  xmm1",
        "movdqu [rsp + 32],  xmm2",
        "movdqu [rsp + 48],  xmm3",
        "movdqu [rsp + 64],  xmm4",
        "movdqu [rsp + 80],  xmm5",
        "movdqu [rsp + 96],  xmm6",
        "movdqu [rsp + 112], xmm7",
        "movdqu [rsp + 128], xmm8",
        "movdqu [rsp + 144], xmm9",
        "movdqu [rsp + 160], xmm10",
        "movdqu [rsp + 176], xmm11",
        "movdqu [rsp + 192], xmm12",
        "movdqu [rsp + 208], xmm13",
        "movdqu [rsp + 224], xmm14",
        "movdqu [rsp + 240], xmm15",

        // ── call async_preempt2 (yields via mcall → schedule) ───────────────
        //
        // The interrupted RSP is NOT guaranteed to be ≡ 8 (mod 16): an async
        // signal can land at any instruction boundary — mid-argument-push,
        // in a prologue, in a leaf function — where RSP may be at either
        // parity.  The System V ABI requires RSP%16 == 0 at every `call`
        // site; violating it makes any aligned SSE spill (`movaps`) in
        // async_preempt2 or its callees fault or corrupt.  Align dynamically
        // and stash the pre-alignment RSP in STACK memory (not a register):
        // stack words survive a copystack relocation because the conservative
        // scan adjusts them, whereas a callee-saved register carried across
        // the mcall suspension would go stale if the stack grows while the
        // goroutine is parked.
        "mov rax, rsp",     // rax = frame base (user RAX already saved above)
        "and rsp, -16",     // drop 0 or 8 bytes — RSP%16 == 0
        "push rax",         // save frame base on the aligned stack (RSP%16 == 8)
        "sub rsp, 8",       // restore call-site alignment   (RSP%16 == 0)
        "call {ap2}",
        "add rsp, 8",
        "pop rax",          // frame base back (adjusted by copystack if grown)
        "mov rsp, rax",     // undo the alignment — RSP = frame base

        // ── restore XMM registers ────────────────────────────────────────────
        "movdqu xmm0,  [rsp + 0]",
        "movdqu xmm1,  [rsp + 16]",
        "movdqu xmm2,  [rsp + 32]",
        "movdqu xmm3,  [rsp + 48]",
        "movdqu xmm4,  [rsp + 64]",
        "movdqu xmm5,  [rsp + 80]",
        "movdqu xmm6,  [rsp + 96]",
        "movdqu xmm7,  [rsp + 112]",
        "movdqu xmm8,  [rsp + 128]",
        "movdqu xmm9,  [rsp + 144]",
        "movdqu xmm10, [rsp + 160]",
        "movdqu xmm11, [rsp + 176]",
        "movdqu xmm12, [rsp + 192]",
        "movdqu xmm13, [rsp + 208]",
        "movdqu xmm14, [rsp + 224]",
        "movdqu xmm15, [rsp + 240]",

        // ── release XMM save area (264 B to match allocation above) ─────────
        "add rsp, 264",

        // ── restore general-purpose registers ────────────────────────────────
        "pop rax",
        "pop rbx",
        "pop rcx",
        "pop rdx",
        "pop rsi",
        "pop rdi",
        "pop r8",
        "pop r9",
        "pop r10",
        "pop r11",
        "pop r12",
        "pop r13",
        "pop r14",
        "pop r15",
        "pop rbp",

        // ── restore RFLAGS ────────────────────────────────────────────────────
        // popfq pops 64-bit RFLAGS from [RSP] and increments RSP by 8.
        // Must come AFTER GPR restores and BEFORE `ret`.
        "popfq",

        // ── return to the original interrupted PC ─────────────────────────────
        // [RSP] holds the original RIP (placed there by the SIGURG handler).
        //
        // `ret 128` (RET imm16) pops the return address and THEN adds 128 to
        // RSP, undoing the red-zone skip applied by redirect_to_async_preempt
        // (which stored the resume PC at rsp − 128 − 8 instead of rsp − 8 so
        // that this trampoline's 392-byte frame cannot clobber the System V
        // red zone of an interrupted leaf function).  After the pop + add,
        // RSP is exactly the interrupted RSP.
        "ret 128",

        ap2 = sym crate::runtime::sched::async_preempt2,
    )
}

/// Win64 variant of [`async_preempt_trampoline`] (non-Windows version above).
///
/// `preempt_m_windows` (in `sched.rs`) injects a call to this function by
/// editing the suspended thread's `CONTEXT`: it stores the resume `RIP` at
/// `rsp − 128 − 8` and points `RIP` here — exactly the same scheme as the Unix
/// `redirect_to_async_preempt` amd64 branch, so the entry invariant
/// (`[RSP]` = resume PC, `RSP % 16 == 8`) and the closing `ret 128` are
/// identical to the Unix body.  Win64 has no red zone, so skipping 128 bytes is
/// merely harmless slack that keeps the trampoline byte-identical to the Unix
/// one apart from the one delta below.
///
/// **The only difference from the Unix body** is the call site: the Win64 ABI
/// requires the caller to reserve **32 bytes of shadow space** above the return
/// address for the callee to spill its register parameters into.  The Unix body
/// reserves none (System V has no shadow space); doing the same here would let
/// `async_preempt2`'s prologue write through the shadow region and clobber the
/// saved frame-base (`push rax`).  We therefore allocate `8 + 32 = 40` bytes
/// (8 to restore call-site alignment, 32 shadow) instead of 8.
///
/// Like the Unix version this over-saves every register Win64 marks
/// callee-saved (RBX/RBP/RSI/RDI/R12–R15, XMM6–15) plus all caller-saved ones,
/// which is exactly what an arbitrary interrupted point requires.
#[unsafe(naked)]
#[cfg(windows)]
pub(crate) unsafe extern "C" fn async_preempt_trampoline() {
    core::arch::naked_asm!(
        // ── save RFLAGS first (see Unix body for the full rationale) ─────────
        "pushfq",                    // RSP % 16 == 0

        // ── save all 15 GPRs (120 B) ─────────────────────────────────────────
        "push rbp",
        "push r15",
        "push r14",
        "push r13",
        "push r12",
        "push r11",
        "push r10",
        "push r9",
        "push r8",
        "push rdi",
        "push rsi",
        "push rdx",
        "push rcx",
        "push rbx",
        "push rax",                  // RSP % 16 == 8

        // ── allocate + save XMM0–15 (264 B: 256 data + 8 pad) ─────────────────
        "sub rsp, 264",              // RSP % 16 == 0
        "movdqu [rsp + 0],   xmm0",
        "movdqu [rsp + 16],  xmm1",
        "movdqu [rsp + 32],  xmm2",
        "movdqu [rsp + 48],  xmm3",
        "movdqu [rsp + 64],  xmm4",
        "movdqu [rsp + 80],  xmm5",
        "movdqu [rsp + 96],  xmm6",
        "movdqu [rsp + 112], xmm7",
        "movdqu [rsp + 128], xmm8",
        "movdqu [rsp + 144], xmm9",
        "movdqu [rsp + 160], xmm10",
        "movdqu [rsp + 176], xmm11",
        "movdqu [rsp + 192], xmm12",
        "movdqu [rsp + 208], xmm13",
        "movdqu [rsp + 224], xmm14",
        "movdqu [rsp + 240], xmm15",

        // ── call async_preempt2 with Win64 shadow space ──────────────────────
        // Dynamically align (the interrupted RSP parity is unknown) and stash
        // the pre-alignment RSP in STACK memory so copystack can relocate it —
        // see the Unix body.  The sole delta is `sub rsp, 40` (8 realign + 32
        // shadow) in place of the Unix `sub rsp, 8`.
        "mov rax, rsp",              // rax = frame base (user RAX already saved)
        "and rsp, -16",              // RSP % 16 == 0
        "push rax",                  // save frame base (RSP % 16 == 8)
        "sub rsp, 40",               // 8 realign + 32 shadow → RSP % 16 == 0
        "call {ap2}",
        "add rsp, 40",
        "pop rax",                   // frame base back (adjusted by copystack)
        "mov rsp, rax",              // undo alignment

        // ── restore XMM0–15 ──────────────────────────────────────────────────
        "movdqu xmm0,  [rsp + 0]",
        "movdqu xmm1,  [rsp + 16]",
        "movdqu xmm2,  [rsp + 32]",
        "movdqu xmm3,  [rsp + 48]",
        "movdqu xmm4,  [rsp + 64]",
        "movdqu xmm5,  [rsp + 80]",
        "movdqu xmm6,  [rsp + 96]",
        "movdqu xmm7,  [rsp + 112]",
        "movdqu xmm8,  [rsp + 128]",
        "movdqu xmm9,  [rsp + 144]",
        "movdqu xmm10, [rsp + 160]",
        "movdqu xmm11, [rsp + 176]",
        "movdqu xmm12, [rsp + 192]",
        "movdqu xmm13, [rsp + 208]",
        "movdqu xmm14, [rsp + 224]",
        "movdqu xmm15, [rsp + 240]",
        "add rsp, 264",

        // ── restore GPRs ─────────────────────────────────────────────────────
        "pop rax",
        "pop rbx",
        "pop rcx",
        "pop rdx",
        "pop rsi",
        "pop rdi",
        "pop r8",
        "pop r9",
        "pop r10",
        "pop r11",
        "pop r12",
        "pop r13",
        "pop r14",
        "pop r15",
        "pop rbp",

        // ── restore RFLAGS, then return to the interrupted PC ────────────────
        "popfq",
        // `ret 128` pops the resume PC and undoes the 128-byte skip applied by
        // preempt_m_windows — identical to the Unix body.
        "ret 128",

        ap2 = sym crate::runtime::sched::async_preempt2,
    )
}