go-lib 0.6.2

// SPDX-License-Identifier: Apache-2.0
//! Goroutine stack allocator and growth machinery — ported from
//! `runtime/stack.go`, `runtime/signal_unix.go`.
//!
//! ## v0.2.0 — dynamic stack growth
//!
//! Each goroutine starts with a 32 KiB stack in release builds, sized to
//! accommodate Rust's panic + libunwind unwind path without needing a
//! reactive grow mid-unwind (which would invalidate libunwind's saved
//! register snapshot).  Debug builds use 16 KiB on Linux and 64 KiB on
//! macOS / Windows to absorb the wider non-optimised frames produced by
//! debug codegen — see [`GOROUTINE_STACK_BYTES`] for the full table and
//! [`STACK_MIN`] for the absolute floor.  The guard page (`PROT_NONE`)
//! immediately below `stack.lo` turns overflows into a `SIGSEGV`
//! (Linux/Windows) or `SIGBUS` (macOS) that the runtime intercepts and
//! recovers from by growing the stack.
//!
//! When the guard page is touched:
//! 1. `sigsegv_handler` identifies the fault as a goroutine stack overflow.
//! 2. It calls `grow_goroutine_stack_from_signal` which:
//!    a. Allocates a new stack double the current size (capped at 1 GiB).
//!    b. Copies the live portion of the old stack to the new one.
//!    c. Adjusts pointer-sized words in the new stack (conservative scan: full
//!    range `[old_guard_lo, old_hi)` for RSP/RBP; guard-page-only range
//!    `[old_guard_lo, old_lo)` for all other GPRs to avoid false-positive
//!    adjustment of heap pointers that coincide with the old stack range).
//!    d. Updates `G.stack`, `G.stackguard0`, and SP in `ucontext_t` (OS retries the instruction).
//! 3. The SIGSEGV handler returns; the OS restores the updated register state;
//!    the faulting instruction is re-executed and now succeeds.
//!
//! ## Layout of each allocation
//!
//! ```text
//! base ──► ┌──────────────────────────────┐
//!          │  guard page  (PROT_NONE)     │  1 × page_size
//!          ├──────────────────────────────┤ ◄── Stack.lo
//!          │                              │
//!          │  execution stack             │  SP starts at hi, grows ↓
//!          │                              │
//!          └──────────────────────────────┘ ◄── Stack.hi
//! ```
//!
//! ## copystack — conservative pointer adjustment
//!
//! Without GC stack maps we scan every pointer-sized word in the live stack
//! region and adjust those that fall within `[old_guard_lo, old_hi)` (the
//! usable old stack plus its guard page).  Return addresses are in the code
//! segment (a completely different address range) and are never mistakenly
//! adjusted.  Integer values that coincidentally equal a stack address are
//! a theoretical false positive but vanishingly rare for the narrow
//! 2–1024 KiB windows used here.

// Mutex guards the SIGSEGV handler's static (Unix) and the stack pool buckets
// (all platforms).
use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering::Relaxed};
use std::sync::Mutex;
use std::sync::OnceLock;

// Unix-only: mmap constants and signal types.
#[cfg(not(windows))]
use libc::{MAP_ANON, MAP_FAILED, MAP_PRIVATE, PROT_NONE, PROT_READ, PROT_WRITE};

// current_g is only needed by the SIGSEGV handler (Unix-only).
#[cfg(not(windows))]
use super::g::current_g;
use super::g::{casgstatus, readgstatus, Stack, GCOPYSTACK, STACK_GUARD, G};

// ---------------------------------------------------------------------------
// Windows: Win32 virtual-memory API (no libc wrappers available)
// ---------------------------------------------------------------------------

#[cfg(windows)]
mod win32 {
    pub const MEM_COMMIT:   u32 = 0x0000_1000;
    pub const MEM_RESERVE:  u32 = 0x0000_2000;
    pub const MEM_RELEASE:  u32 = 0x0000_8000;
    pub const PAGE_READWRITE: u32 = 0x04;
    pub const PAGE_NOACCESS:  u32 = 0x01;

    #[link(name = "kernel32")]
    unsafe extern "system" {
        pub fn VirtualAlloc(
            lpAddress:         *mut u8,
            dwSize:            usize,
            flAllocationType:  u32,
            flProtect:         u32,
        ) -> *mut u8;
        pub fn VirtualFree(
            lpAddress:   *mut u8,
            dwSize:      usize,
            dwFreeType:  u32,
        ) -> i32;
        pub fn VirtualProtect(
            lpAddress:      *mut u8,
            dwSize:         usize,
            flNewProtect:   u32,
            lpflOldProtect: *mut u32,
        ) -> i32;
    }
}

// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------

/// Absolute minimum goroutine stack size (bytes) — matches Go's
/// `stackMin = 2048`.
///
/// **Not** the initial allocation: see [`GOROUTINE_STACK_BYTES`] (release
/// builds use 32 KiB, debug builds 16–64 KiB).  `STACK_MIN` is the lower
/// bound below which the runtime will not allow a stack to shrink — useful
/// when a future stack-shrink pass lands.
#[allow(dead_code)] // reserved for future stack-shrink logic; doc-referenced.
pub(crate) const STACK_MIN: usize = 2 * 1024;

/// Maximum goroutine stack size (bytes). 1 GiB matches Go's `maxstacksize`.
pub(crate) const STACK_MAX: usize = 1024 * 1024 * 1024;

/// Initial stack size for every new goroutine.
///
/// ## Platform table
///
/// | Platform      | Profile | Size  | Notes                                         |
/// |---------------|---------|-------|-----------------------------------------------|
/// | Linux release | release | 32 KiB| Sized for Rust panic + libunwind unwind path  |
/// | Linux AArch64 | release | 32 KiB| Same; page_size = 4 KiB on most kernels       |
/// | macOS release | release | 32 KiB| Guard-page faults raise SIGBUS, not SIGSEGV   |
/// | Linux debug   | debug   | 16 KiB| Smaller for testing, exercises grow path      |
/// | macOS debug   | debug   | 64 KiB| AArch64 CI: 16 KiB pages + wider debug frames |
/// | Windows debug | debug   | 64 KiB| SEH/VEH overhead + 3–5× wider debug frames    |
/// | Windows release| release| 32 KiB| Proactive growth still handles overflow       |
///
/// ## Why 32 KiB and not Go's 2 KiB minimum
///
/// Go's compiler emits per-function `morestack` checks that detect imminent
/// stack exhaustion and call into the runtime to grow the stack BEFORE any
/// frame is committed.  Without that compiler support, we rely on hardware
/// guard pages: one PROT_NONE page below the usable stack.  Any write into
/// the guard page traps, our SIGSEGV/SIGBUS handler grows the stack, and
/// the OS retries the faulting instruction.
///
/// This works for ordinary functions whose prologues allocate at most one
/// page of frame at a time.  It does **not** work for functions whose
/// prologue `sub rsp, N` jumps past the entire guard page in one instruction
/// — the first memory write then lands in unmapped territory below the
/// guard, and the signal handler cannot safely recover (a register holding
/// the overshot address cannot be adjusted without risking heap-pointer
/// false positives; libunwind has already captured a register snapshot
/// pointing into the pre-grow stack that gets invalidated by relocation).
///
/// Rust's panic path on macOS x86-64 hits this case:
/// `_Unwind_RaiseException` alone allocates a ~3 KiB frame, and
/// `unw_getcontext` allocates another ~8 KiB.  Combined with the few KiB of
/// frames already on the stack at panic time, the deepest point of the
/// unwind needs roughly 16 KiB of contiguous stack.  Allocating 32 KiB up
/// front gives libunwind enough headroom that its prologue allocations stay
/// within the usable region and the existing guard-page-based growth
/// machinery handles the rest reactively.
///
/// ## Per-goroutine memory (release builds)
///
/// ```text
/// Platform         Stack   OS guard   G struct   Total
/// ───────────────  ──────  ─────────  ────────── ──────────
/// Linux / Win x86  32 KiB   4 KiB      128 B      ~36 KiB
/// macOS x86-64     32 KiB   4 KiB      128 B      ~36 KiB
/// macOS AArch64    32 KiB  16 KiB      128 B      ~48 KiB
/// ```
///
/// This is significantly more than Go's ~6 KiB per goroutine.  The trade-off
/// is dynamic-growth coverage: Go's compiler-emitted `morestack` prologue
/// makes 2 KiB starts safe, but we lack that mechanism and so must allocate
/// enough up front to survive the deepest single-frame allocation we will
/// encounter (panic unwinding).
// macOS and Windows debug builds use 64 KiB — enough for the wider
// non-optimised frames produced by debug codegen, including the AArch64
// 16 KiB page-size constraint on macOS.
#[cfg(any(all(windows, debug_assertions), all(target_os = "macos", debug_assertions)))]
pub(crate) const GOROUTINE_STACK_BYTES: usize = 64 * 1024;
// Linux debug: 16 KiB — smaller than the release default so the grow path
// stays exercised under tests.
#[cfg(all(debug_assertions, not(any(windows, target_os = "macos"))))]
pub(crate) const GOROUTINE_STACK_BYTES: usize = 16 * 1024;
// All release builds: 32 KiB.  See doc-comment for the rationale (panic-safe
// minimum on macOS x86-64).
#[cfg(not(debug_assertions))]
pub(crate) const GOROUTINE_STACK_BYTES: usize = 32 * 1024;

/// Stack size for each M's g0 (the scheduler stack).
///
/// The scheduler loop (`schedule` → `findrunnable` → `stopm` → locking →
/// `Condvar::wait`) has a deeper call chain than a typical goroutine, and on
/// Windows in debug builds lock operations and system-call trampolines consume
/// 3–5× more stack than on Unix.  The `exitsyscall0_mcall` slow path adds an
/// extra frame level because `schedule()` is called from within
/// `exitsyscall0_mcall`'s frame rather than from the top of g0.  512 KiB
/// provides comfortable headroom across all platforms and build profiles.
/// Go uses the OS thread's native stack (typically 8 MiB) for g0; we use a
/// fixed 512 KiB mmap'd region with the same guard-page layout as a normal
/// goroutine stack.
pub(crate) const G0_STACK_BYTES: usize = 512 * 1024;

// ---------------------------------------------------------------------------
// Page size
// ---------------------------------------------------------------------------

/// Returns the OS page size, queried once and then cached.
///
/// On macOS/AArch64 (Apple Silicon) this is **16 KiB**; on Linux x86-64 and
/// Windows x86-64 it is typically **4 KiB**.
pub(crate) fn page_size() -> usize {
    static PAGE_SIZE: OnceLock<usize> = OnceLock::new();
    *PAGE_SIZE.get_or_init(|| {
        #[cfg(not(windows))]
        {
            let n = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
            assert!(n > 0, "sysconf(_SC_PAGESIZE) returned {n}");
            n as usize
        }
        // Windows x86-64 always uses 4 KiB pages.
        #[cfg(windows)]
        { 4096usize }
    })
}

// ---------------------------------------------------------------------------
// Allocation
// ---------------------------------------------------------------------------

/// Allocate a goroutine stack of exactly `size` usable bytes (+ 1 guard page).
///
/// Returns a [`Stack`] describing the usable region `[lo, hi)`.
///
/// # Errors
/// Returns a static error string on allocation failure.
pub(crate) unsafe fn stack_alloc_size(size: usize) -> Result<Stack, &'static str> {
    debug_assert!(size.is_power_of_two() || size == STACK_MAX,
        "stack_alloc_size: size must be a power of two");
    let ps    = page_size();
    let total = size + ps; // guard page + usable stack

    #[cfg(not(windows))]
    {
        let base = unsafe {
            libc::mmap(
                std::ptr::null_mut(),
                total,
                PROT_READ | PROT_WRITE,
                MAP_ANON | MAP_PRIVATE,
                -1,
                0,
            )
        };
        if base == MAP_FAILED {
            return Err("stack_alloc_size: mmap failed");
        }
        if unsafe { libc::mprotect(base, ps, PROT_NONE) } != 0 {
            unsafe { libc::munmap(base, total) };
            return Err("stack_alloc_size: mprotect guard page failed");
        }
        let base_addr = base as usize;
        Ok(Stack { lo: base_addr + ps, hi: base_addr + total })
    }

    #[cfg(windows)]
    {
        use win32::*;
        let base = unsafe {
            VirtualAlloc(
                std::ptr::null_mut(),
                total,
                MEM_COMMIT | MEM_RESERVE,
                PAGE_READWRITE,
            )
        };
        if base.is_null() {
            return Err("stack_alloc_size: VirtualAlloc failed");
        }
        let mut old_protect: u32 = 0;
        if unsafe { VirtualProtect(base, ps, PAGE_NOACCESS, &mut old_protect) } == 0 {
            unsafe { VirtualFree(base, 0, MEM_RELEASE) };
            return Err("stack_alloc_size: VirtualProtect guard page failed");
        }
        let base_addr = base as usize;
        Ok(Stack { lo: base_addr + ps, hi: base_addr + total })
    }
}

/// Allocate a new goroutine stack of the default initial size.
///
/// Ported from `stackalloc` in `runtime/stack.go`.
pub(crate) unsafe fn stack_alloc() -> Result<Stack, &'static str> {
    unsafe { stack_pool_alloc(GOROUTINE_STACK_BYTES) }
}

/// Allocate a new g0 (scheduler) stack.
///
/// g0 stacks are larger than goroutine stacks because the scheduler call chain
/// (`schedule` → `findrunnable` → locking → park) is deeper.
pub(crate) unsafe fn g0_stack_alloc() -> Result<Stack, &'static str> {
    unsafe { stack_pool_alloc(G0_STACK_BYTES) }
}

// ---------------------------------------------------------------------------
// Deallocation
// ---------------------------------------------------------------------------

/// Free a goroutine stack previously returned by `stack_alloc_size`.
///
/// # Safety
/// `stack` must have been returned by `stack_alloc_size` and must not have
/// been freed before.
pub(crate) unsafe fn stack_free(stack: &Stack) {
    let ps   = page_size();
    let base = (stack.lo - ps) as *mut u8;

    #[cfg(not(windows))]
    {
        let total = (stack.hi - stack.lo) + ps;
        unsafe { libc::munmap(base as *mut libc::c_void, total) };
    }

    #[cfg(windows)]
    {
        use win32::{MEM_RELEASE, VirtualFree};
        // VirtualFree with MEM_RELEASE requires dwSize = 0.
        unsafe { VirtualFree(base, 0, MEM_RELEASE) };
    }
}

// ---------------------------------------------------------------------------
// Stack pool — size-classed free list of mmap'd stacks
// ---------------------------------------------------------------------------
// Decoupled from the G-descriptor free pool (`G_FREE` in `sched.rs`), this is
// the port of Go's `stackpool` / `stackLarge`: a cache of freed goroutine and
// g0 stacks, keyed by size class, that lets goroutine creation and stack
// growth recycle an existing mmap instead of paying an `mmap`+`mprotect` /
// `munmap` syscall pair every time.  It exists to **bound RSS and syscall
// churn** under bursty goroutine workloads.
//
// ## Async-signal-safety — why the signal path does NOT use the pool
//
// The reactive growth path (`sigsegv_handler` → `try_grow_stack_from_signal`
// → `newstack`) runs in an async-signal context and therefore CANNOT take the
// pool's `Mutex` — a fault that interrupts a thread already holding a bucket
// lock would deadlock that thread against itself.  `newstack` keeps calling
// the raw `stack_alloc_size` / `stack_free` primitives (effectively `mmap` /
// `munmap`, which are safe enough in that context, exactly as before).  Only
// the **non-signal** callers — fresh goroutine creation, g0 allocation, the
// proactive `grow_stack_if_needed` checkpoint (runs on g0), M teardown, and
// the gFree stack handoff — route through the pool.  The two sets of callers
// mix freely: every stack is a uniform `mmap(size + guard_page)` region
// regardless of which path created or freed it.
//
// Each pooled bucket is wrapped in `m_lock()` like the other scheduler-internal
// mutexes so an async-preemption SIGURG cannot migrate the thread mid-`MutexGuard`.

/// Smallest power-of-two size class the pool retains: `1 << 14` = 16 KiB, the
/// smallest real goroutine stack (Linux debug `GOROUTINE_STACK_BYTES`).
const POOL_MIN_CLASS: u32 = 14;
/// Largest power-of-two size class the pool retains: `1 << 21` = 2 MiB.  Larger
/// stacks (grown deep, up to `STACK_MAX`) are freed directly rather than hoarded
/// — bounding the per-entry footprint of the idle cache.
const POOL_MAX_CLASS: u32 = 21;
/// Number of buckets, one per power-of-two exponent in `[MIN, MAX]`.
const NUM_POOL_CLASSES: usize = (POOL_MAX_CLASS - POOL_MIN_CLASS + 1) as usize;
/// Soft cap on the total bytes the idle pool may retain across all classes.
/// A `stack_pool_free` that would push past this unmaps instead of caching, so
/// the pool's contribution to steady-state RSS never exceeds ~this value.
const MAX_POOLED_BYTES: usize = 64 * 1024 * 1024;

/// Per-class free lists of base addresses (`Stack.lo`).  Indexed by
/// `exponent − POOL_MIN_CLASS`.  Addresses are stored as `usize` because
/// `*mut` is not `Send`; the guard page below each `lo` stays `PROT_NONE`
/// throughout, so a pooled entry is reusable as-is with no re-`mprotect`.
static STACK_POOL: [Mutex<Vec<usize>>; NUM_POOL_CLASSES] =
    [const { Mutex::new(Vec::new()) }; NUM_POOL_CLASSES];

/// Running total of bytes currently parked in [`STACK_POOL`] (the usable region
/// of each entry; the guard page is excluded, matching the size key).
static POOLED_BYTES: AtomicUsize = AtomicUsize::new(0);

/// Whether the stack pool is disabled (`GOLIB_STACKPOOL_OFF=1`).  Ablation lever
/// — independent of the G-descriptor pool's `GOLIB_GPOOL_OFF` — for
/// A/B-bisecting any pooling regression: when set, alloc always `mmap`s fresh
/// and free always `munmap`s.  `u8::MAX` sentinel = not yet read.
static STACKPOOL_OFF: AtomicU8 = AtomicU8::new(u8::MAX);

#[inline]
fn stackpool_off() -> bool {
    let mut v = STACKPOOL_OFF.load(Relaxed);
    if v == u8::MAX {
        v = match std::env::var("GOLIB_STACKPOOL_OFF") {
            Ok(s) if s == "1" => 1,
            _ => 0,
        };
        STACKPOOL_OFF.store(v, Relaxed);
    }
    v == 1
}

/// Map a usable stack size to its pool bucket index, or `None` if the size is
/// outside the pooled range (`< 16 KiB`, `> 2 MiB`, or not a power of two).
#[inline]
fn pool_class(size: usize) -> Option<usize> {
    if !size.is_power_of_two() {
        return None;
    }
    let exp = size.trailing_zeros();
    if !(POOL_MIN_CLASS..=POOL_MAX_CLASS).contains(&exp) {
        return None;
    }
    Some((exp - POOL_MIN_CLASS) as usize)
}

/// Allocate a stack of `size` usable bytes, reusing a pooled one of the same
/// size class when available and otherwise falling back to [`stack_alloc_size`].
///
/// This is the pooled entry point used by every **non-signal** caller; the
/// async-signal growth path keeps calling [`stack_alloc_size`] directly (see the
/// module-level note on signal-safety).
///
/// # Safety
/// Same contract as [`stack_alloc_size`]: the returned [`Stack`] must eventually
/// be released via [`stack_pool_free`] (or [`stack_free`]).
pub(crate) unsafe fn stack_pool_alloc(size: usize) -> Result<Stack, &'static str> {
    if !stackpool_off() && let Some(cls) = pool_class(size) {
        // Scope the bucket lock + M-pin tightly; never hold across the mmap
        // fallback below.
        let popped = {
            let _pin = super::m::m_lock();
            STACK_POOL[cls].lock().unwrap().pop()
        };
        if let Some(lo) = popped {
            POOLED_BYTES.fetch_sub(size, Relaxed);
            // Guard page below `lo` is still PROT_NONE from the original
            // mapping; the entry is reusable verbatim.
            return Ok(Stack { lo, hi: lo + size });
        }
    }
    unsafe { stack_alloc_size(size) }
}

/// Return a stack to the pool for reuse, or unmap it if its size is outside the
/// pooled range or the pool is already at [`MAX_POOLED_BYTES`].
///
/// The pooled entry retains its guard page (`PROT_NONE`) intact; only the base
/// address is recorded.
///
/// # Safety
/// `stack` must have been returned by [`stack_alloc_size`] / [`stack_pool_alloc`]
/// and must not be freed or used again after this call.
pub(crate) unsafe fn stack_pool_free(stack: &Stack) {
    let size = stack.hi - stack.lo;
    if !stackpool_off() && let Some(cls) = pool_class(size) {
        // Reserve the bytes first; if that would exceed the soft cap, undo
        // and unmap instead.  The check races benignly under concurrency —
        // the cap is a soft RSS bound, not a hard invariant.
        let prev = POOLED_BYTES.fetch_add(size, Relaxed);
        if prev + size <= MAX_POOLED_BYTES {
            let _pin = super::m::m_lock();
            STACK_POOL[cls].lock().unwrap().push(stack.lo);
            return;
        }
        POOLED_BYTES.fetch_sub(size, Relaxed);
    }
    unsafe { stack_free(stack) };
}

// ---------------------------------------------------------------------------
// Stack growth — newstack / copystack
// ---------------------------------------------------------------------------

/// Double the goroutine's stack (up to STACK_MAX) and resume it.
///
/// Called from the SIGSEGV handler when a guard page fault is detected.
/// On return the goroutine's stack fields are updated; the caller must also
/// update the interrupted `RSP/SP` in the platform `ucontext_t`.
///
/// Returns the delta applied to all adjusted pointers.
///
/// # Safety
/// Must be called from a signal handler context; `gp` must be the goroutine
/// whose guard page was touched.
///
/// **Not compiled on Windows** — Windows has no POSIX SIGSEGV mechanism.
/// Proactive growth via `grow_stack_if_needed` handles Windows instead.
#[cfg(not(windows))]
pub(crate) unsafe fn newstack(gp: *mut G) -> isize {
    let old_stack = Stack {
        lo: unsafe { (*gp).stack.lo },
        hi: unsafe { (*gp).stack.hi },
    };
    let old_size = old_stack.hi - old_stack.lo;

    if old_size >= STACK_MAX {
        // Stack already at maximum — this is a genuine overflow.
        eprintln!("goroutine stack overflow: stack size {old_size} >= STACK_MAX ({STACK_MAX})");
        unsafe { libc::abort() };
    }

    let new_size = (old_size * 2).min(STACK_MAX);
    let new_stack = unsafe {
        stack_alloc_size(new_size).expect("newstack: failed to allocate new goroutine stack")
    };

    // Copy live portion and adjust pointers.
    let delta = unsafe { copystack(gp, &old_stack, &new_stack) };

    // Update G's stack bookkeeping.
    unsafe {
        (*gp).stack       = Stack { lo: new_stack.lo, hi: new_stack.hi };
        (*gp).stackguard0 = new_stack.lo + STACK_GUARD;
    }

    // Free the old stack.
    unsafe { stack_free(&old_stack) };

    delta
}

/// Copy the live portion of a goroutine's stack to a new allocation and
/// apply a conservative pointer-adjustment scan.
///
/// Returns the delta (`new_stack.lo as isize - old_stack.lo as isize`).
///
/// # Conservative pointer adjustment
///
/// Every pointer-sized word in the copied region that falls within
/// `[old_lo, old_hi)` is incremented by `delta`.  Words outside that range
/// (code pointers / heap pointers / integers) are unchanged.
///
/// # Safety
/// `old_stack` must be the goroutine's current live stack; `new_stack` must
/// be freshly allocated with at least the same usable size.
unsafe fn copystack(gp: *mut G, old_stack: &Stack, new_stack: &Stack) -> isize {
    // Bracket the copy with GCOPYSTACK so a future GC scanner skips this G
    // while its stack is in a half-copied state.
    // GRUNNING → GCOPYSTACK → GRUNNING  (matches Go's casgcopystack protocol).
    let old_status = unsafe { readgstatus(gp) };
    unsafe { casgstatus(gp, old_status, GCOPYSTACK) };

    let old_lo = old_stack.lo;
    let old_hi = old_stack.hi;
    let new_lo = new_stack.lo;
    let new_hi = new_stack.hi;
    let _old_size = old_hi - old_lo;
    let new_size = new_hi - new_lo;

    // The live stack occupies the top portion (stacks grow down).
    // Saved SP tells us how far down the goroutine has grown.
    // If sched.sp is 0 (goroutine not yet started), treat the whole stack as live.
    let saved_sp = unsafe { (*gp).sched.sp };
    let live_start_old = if saved_sp != 0 && saved_sp >= old_lo && saved_sp < old_hi {
        saved_sp
    } else {
        old_lo // treat as fully live for safety
    };

    // Offset of the live portion from old_hi.
    let live_bytes = old_hi - live_start_old;

    // Corresponding start in the new stack (preserving relative position from hi).
    let live_start_new = new_hi - live_bytes;

    // Bounds check: the new stack must be large enough.
    debug_assert!(
        new_size >= live_bytes,
        "copystack: new stack ({new_size} B) too small for live region ({live_bytes} B)"
    );

    // Copy live bytes from old → new.
    unsafe {
        std::ptr::copy_nonoverlapping(
            live_start_old as *const u8,
            live_start_new as *mut u8,
            live_bytes,
        );
    }

    // Delta is the displacement applied to old-stack pointers to produce
    // new-stack pointers.  The live region is copied relative to `hi`
    // (stacks grow downward), so the correct displacement is new_hi − old_hi,
    // NOT new_lo − old_lo.  Using new_lo − old_lo would be wrong when the two
    // stacks differ in size (the new stack is larger), which is always the
    // case during growth.  This matches Go runtime's adjinfo.delta calculation:
    //   delta = new.hi - old.hi
    let delta: isize = new_hi as isize - old_hi as isize;

    // Conservative scan: adjust any pointer-sized word in the new live region
    // that falls within [old_guard_lo, old_hi).
    //
    // We extend the lower bound from `old_lo` to `old_lo − page_size` (the
    // start of the old guard page) so that we also adjust words that hold
    // addresses the goroutine computed via a large negative offset from the
    // frame pointer — e.g. `lea rdi, [rbp − 4168]` where the result lands in
    // the guard page.  Those addresses are passed as arguments to functions
    // (e.g. `memset`) and must be relocated to the equivalent position in the
    // new, larger stack.
    let old_guard_lo = old_lo.saturating_sub(page_size());
    let mut addr = live_start_new;
    let word = std::mem::size_of::<usize>();
    while addr + word <= new_hi {
        let val = unsafe { *(addr as *const usize) };
        if val >= old_guard_lo && val < old_hi {
            unsafe { *(addr as *mut usize) = ((val as isize) + delta) as usize };
        }
        addr += word;
    }

    // Update G's saved registers that point into the old stack.
    unsafe {
        let sp = (*gp).sched.sp;
        if sp >= old_lo && sp < old_hi {
            (*gp).sched.sp = ((sp as isize) + delta) as usize;
        }
        let bp = (*gp).sched.bp;
        if bp >= old_lo && bp < old_hi {
            (*gp).sched.bp = ((bp as isize) + delta) as usize;
        }
        // gopark_commit's unlock argument may point into the goroutine's own
        // stack (e.g. a select unlock descriptor); relocate it like sp/bp.
        let pua = (*gp).park_unlock_arg as usize;
        if pua >= old_lo && pua < old_hi {
            (*gp).park_unlock_arg = ((pua as isize) + delta) as *mut u8;
        }
    }

    // Restore the original status: GCOPYSTACK → old_status.
    unsafe { casgstatus(gp, GCOPYSTACK, old_status) };

    delta
}

// ---------------------------------------------------------------------------
// SIGSEGV handler — guard page detection and stack growth (Unix only)
// ---------------------------------------------------------------------------
// Windows does not have POSIX signals; guard-page faults are handled instead
// by the proactive `grow_stack_if_needed` checkpoint in the scheduler.

/// Previous SIGSEGV handler (chained if fault is not a stack overflow).
#[cfg(not(windows))]
static PREV_SIGSEGV: Mutex<Option<libc::sigaction>> = Mutex::new(None);

/// Install the runtime's SIGSEGV handler for goroutine stack guard pages.
///
/// If the faulting address falls in the guard page of the current goroutine's
/// stack, the handler grows the stack and updates the interrupt context so the
/// retry succeeds.  All other SIGSEGVs are forwarded to the previous handler.
///
/// # Safety
/// Call once from the main initialisation path (inside `schedinit`).
#[cfg(not(windows))]
pub(crate) unsafe fn install_sigsegv_handler() {
    let mut sa: libc::sigaction = unsafe { std::mem::zeroed() };
    sa.sa_sigaction = sigsegv_handler as *const () as usize;
    // sa_flags is c_ulong on Linux and c_int on macOS; `as _` lets Rust infer the right type.
    sa.sa_flags     = (libc::SA_SIGINFO | libc::SA_ONSTACK | libc::SA_RESTART) as _;
    unsafe { libc::sigemptyset(&mut sa.sa_mask) };

    let mut old: libc::sigaction = unsafe { std::mem::zeroed() };
    let ret = unsafe { libc::sigaction(libc::SIGSEGV, &sa, &mut old) };
    assert_eq!(ret, 0, "install_sigsegv_handler: sigaction failed");

    *PREV_SIGSEGV.lock().unwrap() = Some(old);
}

/// Shared guard-page fault handler — called from both SIGSEGV and SIGBUS handlers.
///
/// On Linux, `mprotect(PROT_NONE)` guard-page faults raise `SIGSEGV`.
/// On macOS, the same access raises `SIGBUS` (permission violation on a mapped
/// page) rather than `SIGSEGV` (unmapped address).  Both signals use this
/// identical detection-and-growth path.
///
/// ## Why we zero `gp.sched.sp` before copying
///
/// `gp.sched.sp` holds the SP that was saved at the goroutine's last
/// scheduling point (the most recent `mcall` or `gopark`).  While the goroutine
/// is *running*, any frames pushed since that point are below the saved SP and
/// are not reflected in `gp.sched.sp`.  `copystack` uses `gp.sched.sp` as the
/// start of the live region; if it is stale, only the top few bytes of the
/// stack would be copied and the actual live frames would be lost.
///
/// Setting `gp.sched.sp = 0` before calling `newstack` triggers `copystack`'s
/// "treat entire stack as live" fallback (`saved_sp == 0` → `live_start =
/// stack.lo`), which is always correct: copying a few extra dead bytes is safe,
/// but missing live frames is not.
///
/// ## Why we adjust all general-purpose registers
///
/// A function that overflows the guard page may compute a stack address via a
/// large negative offset from its frame pointer and pass that address to a
/// library call (e.g. `lea rdi, [rbp−4168]; call memset`) before allocating
/// the frame with `sub rsp, 4168`.  When `memset` faults at the destination
/// address, RDI holds a value in the **guard page** (below `old_lo`).
/// Updating only RSP/RBP leaves RDI pointing to the freed guard page; the
/// retry faults again as SIGSEGV on the now-unmapped page.
///
/// `update_sp_in_context` therefore adjusts **all** general-purpose registers
/// whose values fall in `[old_lo − page_size, old_hi)` — the usable old stack
/// plus the guard page — by `delta`, relocating every potentially stale stack
/// reference to the equivalent location in the new, larger stack.
///
/// Returns `true` if the fault was a goroutine stack overflow and has been
/// handled (caller should return from the signal handler); `false` if the fault
/// is unrelated to a guard page and the caller should chain to its normal path.
#[cfg(not(windows))]
pub(crate) unsafe fn try_grow_stack_from_signal(
    fault_addr: usize,
    ctx:        *mut libc::c_void,
) -> bool {
    let gp = current_g();
    if gp.is_null() { return false; }

    let stack_lo = unsafe { (*gp).stack.lo };
    let stack_hi = unsafe { (*gp).stack.hi };
    let guard_lo = stack_lo - page_size();
    let guard_hi = stack_lo;

    if fault_addr < guard_lo || fault_addr >= guard_hi {
        return false; // not a goroutine stack overflow
    }

    // Force copystack to copy the full old stack (see doc comment above).
    unsafe { (*gp).sched.sp = 0 };

    // Save old bounds before newstack() updates gp.stack.
    let old_lo = stack_lo;
    let old_hi = stack_hi;

    // Grow the stack; adjust all GPRs that hold old-stack references so that
    // the OS-retried faulting instruction succeeds on the new stack.
    let delta = unsafe { newstack(gp) };
    unsafe { update_sp_in_context(ctx, old_lo, old_hi, delta) };
    true
}

/// SIGSEGV handler: detect goroutine guard page faults and grow the stack.
#[cfg(not(windows))]
unsafe extern "C" fn sigsegv_handler(
    sig:  libc::c_int,
    info: *mut libc::siginfo_t,
    ctx:  *mut libc::c_void,
) {
    let fault_addr = unsafe { (*info).si_addr() } as usize;
    if unsafe { try_grow_stack_from_signal(fault_addr, ctx) } {
        return; // handler return → OS retries the faulting instruction
    }

    // Not a stack fault — print async-signal-safe diagnostics, then chain.
    // Mirrors the SIGBUS handler in sched.rs: write(2) only, no allocation.
    {
        #[inline(always)]
        unsafe fn sig_write(msg: &[u8]) {
            unsafe { libc::write(2, msg.as_ptr() as *const libc::c_void, msg.len()) };
        }
        #[inline(always)]
        unsafe fn sig_hex(label: &[u8], val: u64) {
            unsafe { sig_write(label) };
            const H: &[u8] = b"0123456789abcdef";
            let mut buf = [b'0'; 19];
            buf[0] = b'0'; buf[1] = b'x';
            for i in 0..16usize { buf[17 - i] = H[((val >> (i * 4)) & 0xf) as usize]; }
            buf[18] = b'\n';
            unsafe { sig_write(&buf) };
        }
        unsafe {
            sig_write(b"[go-lib SIGSEGV] non-stack fault\n");
            sig_hex(b"[go-lib SIGSEGV] fault_addr = ", fault_addr as u64);
        }
        let gp = current_g();
        if !gp.is_null() {
            unsafe {
                sig_hex(b"[go-lib SIGSEGV] g.stack.lo = ", (*gp).stack.lo as u64);
                sig_hex(b"[go-lib SIGSEGV] g.stack.hi = ", (*gp).stack.hi as u64);
            }
        }
        #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
        if !ctx.is_null() {
            unsafe {
                let uc = ctx as *mut libc::ucontext_t;
                let ss = &(*(*uc).uc_mcontext).__ss;
                sig_hex(b"[go-lib SIGSEGV] PC = ", ss.__pc);
                sig_hex(b"[go-lib SIGSEGV] LR = ", ss.__lr);
                sig_hex(b"[go-lib SIGSEGV] SP = ", ss.__sp);
                sig_hex(b"[go-lib SIGSEGV] FP = ", ss.__fp);
                sig_hex(b"[go-lib SIGSEGV] x19 = ", ss.__x[19]);
                sig_hex(b"[go-lib SIGSEGV] x20 = ", ss.__x[20]);
                sig_hex(b"[go-lib SIGSEGV] x21 = ", ss.__x[21]);
                sig_hex(b"[go-lib SIGSEGV] x22 = ", ss.__x[22]);
            }
        }
        #[cfg(all(target_os = "macos", target_arch = "x86_64"))]
        if !ctx.is_null() {
            unsafe {
                let uc = ctx as *mut libc::ucontext_t;
                let ss = &(*(*uc).uc_mcontext).__ss;
                sig_hex(b"[go-lib SIGSEGV] RIP = ", ss.__rip);
                sig_hex(b"[go-lib SIGSEGV] RSP = ", ss.__rsp);
            }
        }
        #[cfg(all(target_os = "linux", target_arch = "x86_64"))]
        if !ctx.is_null() {
            unsafe {
                let uc = ctx as *mut libc::ucontext_t;
                let gregs = &(*uc).uc_mcontext.gregs;
                sig_hex(b"[go-lib SIGSEGV] RIP = ", gregs[libc::REG_RIP as usize] as u64);
                sig_hex(b"[go-lib SIGSEGV] RSP = ", gregs[libc::REG_RSP as usize] as u64);
                sig_hex(b"[go-lib SIGSEGV] RBP = ", gregs[libc::REG_RBP as usize] as u64);
                sig_hex(b"[go-lib SIGSEGV] RBX = ", gregs[libc::REG_RBX as usize] as u64);
                sig_hex(b"[go-lib SIGSEGV] R12 = ", gregs[libc::REG_R12 as usize] as u64);
                sig_hex(b"[go-lib SIGSEGV] R13 = ", gregs[libc::REG_R13 as usize] as u64);
                sig_hex(b"[go-lib SIGSEGV] R14 = ", gregs[libc::REG_R14 as usize] as u64);
                sig_hex(b"[go-lib SIGSEGV] R15 = ", gregs[libc::REG_R15 as usize] as u64);
            }
        }
        let mp = super::m::current_m();
        if !mp.is_null() {
            let g0 = unsafe { (*mp).g0 };
            if !g0.is_null() {
                unsafe {
                    sig_hex(b"[go-lib SIGSEGV] g0.stack.lo = ", (*g0).stack.lo as u64);
                    sig_hex(b"[go-lib SIGSEGV] g0.stack.hi = ", (*g0).stack.hi as u64);
                }
            }
        }
    }

    let prev = *PREV_SIGSEGV.lock().unwrap();
    match prev {
        Some(old) if old.sa_sigaction != libc::SIG_DFL
                  && old.sa_sigaction != libc::SIG_IGN => {
            // Call the previous handler.
            type SaFn = unsafe extern "C" fn(libc::c_int, *mut libc::siginfo_t, *mut libc::c_void);
            let f: SaFn = unsafe { std::mem::transmute(old.sa_sigaction) };
            unsafe { f(sig, info, ctx) };
        }
        _ => {
            // Default action: terminate with SIGSEGV.
            unsafe { libc::raise(libc::SIGSEGV) };
        }
    }
}

/// Return the number of bytes that SP was **pre-decremented** by the faulting
/// AArch64 instruction at `pc`, or 0 if the instruction does not pre-decrement.
///
/// On AArch64, pre-indexed store instructions (e.g. `stp x29, x30, [sp, #-16]!`)
/// commit the base-register writeback before the data-abort fires on most
/// implementations.  Retrying the instruction would decrement SP a second time,
/// corrupting the caller's frame.  Adding the pre-decrement back to SP in the
/// ucontext before the retry causes the instruction to land at the correct
/// location.
///
/// On x86-64, `push rXX` does NOT commit the RSP decrement when the store
/// page-faults (the push is architecturally atomic).  This function is therefore
/// only compiled for AArch64 targets.
#[cfg(all(not(windows), target_arch = "aarch64"))]
unsafe fn sp_predecrement_at_pc(pc: usize) -> usize {
    if pc == 0 { return 0; }
    // AArch64 instructions are always 4 bytes, little-endian.
    let instr = unsafe { *(pc as *const u32) };

    // STP 64-bit GPR pre-indexed: bits [31:22] = 0x2A6, imm7 scaled ×8.
    if (instr >> 22) & 0x3FF == 0x2A6 {
        let imm7 = (((instr >> 15) & 0x7F) as i32) << 25 >> 25;
        if imm7 < 0 { return (-imm7 * 8) as usize; }
    }
    // STP 32-bit GPR pre-indexed: bits [31:22] = 0x0A6, imm7 scaled ×4.
    if (instr >> 22) & 0x3FF == 0x0A6 {
        let imm7 = (((instr >> 15) & 0x7F) as i32) << 25 >> 25;
        if imm7 < 0 { return (-imm7 * 4) as usize; }
    }
    // STR 64-bit pre-indexed: bits [31:21] = 0x7C0, bits [11:10] = 3.
    if (instr >> 21) & 0x7FF == 0x7C0 && (instr >> 10) & 3 == 3 {
        let imm9 = (((instr >> 12) & 0x1FF) as i32) << 23 >> 23;
        if imm9 < 0 { return (-imm9) as usize; }
    }
    0
}

/// Update the interrupted register state saved in the signal `ucontext_t` after
/// a goroutine stack has been grown.
///
/// ## Two-range register adjustment
///
/// A function that overflows the guard page may compute a stack address via a
/// large negative offset from the frame pointer — e.g.
/// `lea rdi, [rbp − 4168]; call memset` — so RDI holds an address in the
/// guard page (below `old_lo`).  Updating only RSP/RBP leaves RDI pointing to
/// the now-unmapped guard page; the retry faults again immediately.
///
/// We scan the register file in two groups:
///
/// 1. **SP and FP only** (RSP, RBP / SP, x29):
///    full range `[old_guard_lo, old_hi)`.  These definitively hold stack
///    frame-chain pointers.
///
/// 2. **All other GPRs** (callee-saved RBX/R12–R15 + caller-saved/argument
///    RAX–RDX/RSI/RDI/R8–R11 / x0–x28):
///    narrow range `[old_guard_lo, old_lo)` — the guard page only.  This
///    handles the `lea rdi, [rbp−N]` overflow-into-guard-page pattern while
///    avoiding false-positive adjustments of heap pointers.  Callee-saved
///    registers (RBX, R12–R15) commonly hold heap pointers (Vec/HashMap
///    data pointers, Arc data, etc.) whose values can coincide with the
///    usable-stack address range at scale; adjusting them in the full range
///    caused channel-buffer discriminant flips observed at WORKERS=75 000
///    (fixed in PR #23).
///
/// Platform-specific: Linux x86-64, Linux AArch64, macOS x86-64, macOS AArch64.
#[cfg(not(windows))]
unsafe fn update_sp_in_context(
    ctx:    *mut libc::c_void,
    old_lo: usize,
    old_hi: usize,
    delta:  isize,
) {
    // old_guard_lo: start of the old guard page (PROT_NONE region).
    // No valid heap allocation can point into [old_guard_lo, old_lo).
    let old_guard_lo = old_lo.saturating_sub(page_size());

    /// Adjust `val` by `delta` iff it falls within `[lo, hi)`.
    #[inline(always)]
    fn adj(val: u64, lo: usize, hi: usize, delta: isize) -> u64 {
        let v = val as usize;
        if v >= lo && v < hi { (v as isize + delta) as u64 } else { val }
    }

    #[cfg(all(target_os = "linux", target_arch = "x86_64"))]
    unsafe {
        use libc::{REG_RAX,REG_RBX,REG_RCX,REG_RDX,REG_RSI,REG_RDI,
                   REG_RBP,REG_RSP,REG_R8,REG_R9,REG_R10,REG_R11,
                   REG_R12,REG_R13,REG_R14,REG_R15};
        let mc = &mut (*(ctx as *mut libc::ucontext_t)).uc_mcontext;
        // SP + FP only: full range — these definitely point into the old stack.
        for reg in [REG_RSP, REG_RBP] {
            let v = mc.gregs[reg as usize] as u64;
            mc.gregs[reg as usize] = adj(v, old_guard_lo, old_hi, delta) as libc::greg_t;
        }
        // Callee-saved (RBX, R12-R15) + caller-saved/argument: guard-page only.
        // These registers commonly hold heap pointers (esp. RBX/R12-R15 for
        // Vec/HashMap data pointers); adjusting them in the usable-stack
        // range causes false positives at scale — observed channel-buffer
        // discriminant flip after stack growth with 75k goroutines (PR #22+).
        for reg in [REG_RBX, REG_R12, REG_R13, REG_R14, REG_R15,
                    REG_RAX, REG_RCX, REG_RDX, REG_RSI, REG_RDI,
                    REG_R8,  REG_R9,  REG_R10, REG_R11] {
            let v = mc.gregs[reg as usize] as u64;
            mc.gregs[reg as usize] = adj(v, old_guard_lo, old_lo, delta) as libc::greg_t;
        }
    }

    #[cfg(all(target_os = "linux", target_arch = "aarch64"))]
    unsafe {
        let mc = &mut (*(ctx as *mut libc::ucontext_t)).uc_mcontext;
        // SP and FP (x29): full range — definite stack pointers.
        mc.sp       = adj(mc.sp,       old_guard_lo, old_hi, delta);
        mc.regs[29] = adj(mc.regs[29], old_guard_lo, old_hi, delta);
        // All other GPRs: guard-page only (avoid heap-pointer false positives).
        for i in 0..=28usize {
            if i == 29 { continue; }
            mc.regs[i] = adj(mc.regs[i], old_guard_lo, old_lo, delta);
        }
        // AArch64 pre-indexed stores (e.g. `stp x29,x30,[sp,#-16]!`) commit
        // the base-register update even on a data-abort on most implementations.
        // Undo the pre-decrement so the retry instruction lands correctly.
        let correction = sp_predecrement_at_pc(mc.pc as usize) as u64;
        if correction != 0 { mc.sp += correction; }
    }

    #[cfg(all(target_os = "macos", target_arch = "x86_64"))]
    unsafe {
        let ss = &mut (*(*  (ctx as *mut libc::ucontext_t)).uc_mcontext).__ss;
        // SP + FP: full range — these definitely point into the old stack.
        ss.__rsp = adj(ss.__rsp, old_guard_lo, old_hi, delta);
        ss.__rbp = adj(ss.__rbp, old_guard_lo, old_hi, delta);
        // Callee-saved (RBX, R12-R15) + caller-saved: guard-page only.
        // These registers commonly hold heap pointers (esp. RBX/R12-R15 for
        // Vec/HashMap data pointers); adjusting them in the usable-stack
        // range causes false positives at scale — observed channel-buffer
        // discriminant flip after stack growth with 75k goroutines (PR #22+).
        ss.__rbx = adj(ss.__rbx, old_guard_lo, old_lo, delta);
        ss.__r12 = adj(ss.__r12, old_guard_lo, old_lo, delta);
        ss.__r13 = adj(ss.__r13, old_guard_lo, old_lo, delta);
        ss.__r14 = adj(ss.__r14, old_guard_lo, old_lo, delta);
        ss.__r15 = adj(ss.__r15, old_guard_lo, old_lo, delta);
        ss.__rax = adj(ss.__rax, old_guard_lo, old_lo, delta);
        ss.__rcx = adj(ss.__rcx, old_guard_lo, old_lo, delta);
        ss.__rdx = adj(ss.__rdx, old_guard_lo, old_lo, delta);
        ss.__rsi = adj(ss.__rsi, old_guard_lo, old_lo, delta);
        ss.__rdi = adj(ss.__rdi, old_guard_lo, old_lo, delta);
        ss.__r8  = adj(ss.__r8,  old_guard_lo, old_lo, delta);
        ss.__r9  = adj(ss.__r9,  old_guard_lo, old_lo, delta);
        ss.__r10 = adj(ss.__r10, old_guard_lo, old_lo, delta);
        ss.__r11 = adj(ss.__r11, old_guard_lo, old_lo, delta);
        // Note: x86-64 `push rXX` that page-faults does NOT commit the RSP
        // decrement (the push is atomic).  No SP correction is needed.
    }

    #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
    unsafe {
        let ss = &mut (*(*(ctx as *mut libc::ucontext_t)).uc_mcontext).__ss;
        // SP and FP (__fp = x29): full range — definite stack pointers.
        ss.__sp = adj(ss.__sp, old_guard_lo, old_hi, delta);
        ss.__fp = adj(ss.__fp, old_guard_lo, old_hi, delta);
        // All other GPRs (x0–x28): guard-page only.  Avoids false positives
        // when a register coincidentally holds a heap pointer that falls in
        // the old stack range — observed channel-buffer corruption at 75k
        // goroutines when callee-saved x19–x28 were adjusted in the full
        // range (PR #22 follow-up).
        for i in 0..=28usize {
            ss.__x[i] = adj(ss.__x[i], old_guard_lo, old_lo, delta);
        }
        // AArch64 pre-indexed stores commit SP before the data-abort.
        let correction = sp_predecrement_at_pc(ss.__pc as usize) as u64;
        if correction != 0 { ss.__sp += correction; }
    }
}

// ---------------------------------------------------------------------------
// Grow stack at scheduler re-entry (checkpoint growth)
// ---------------------------------------------------------------------------

/// Check whether `gp`'s saved stack pointer is within `STACK_GUARD` bytes of
/// the guard page, and if so grow the stack proactively before resuming.
///
/// Called from `execute` in `sched.rs` (on g0, before every `gogo` call).
/// Handles the case where a goroutine exhausts most of its stack across
/// multiple scheduler quanta and the SIGSEGV handler is about to fire.
///
/// ## Threshold — `STACK_GUARD` (not `2 × STACK_GUARD`)
///
/// We use `STACK_GUARD` (928 bytes) as the low-water mark, matching Go's
/// `stackGuard`.  This leaves exactly one guard zone of headroom — enough
/// for a typical scheduler-call depth — before triggering a doubling.
/// Using `2 × STACK_GUARD` was excessively conservative when the runtime
/// previously started goroutines at `STACK_MIN` (2 KiB): it left only
/// 192 bytes of effective space (`2048 − 1856 = 192`) and would force
/// almost every goroutine to grow on its first scheduling point even when
/// the stack was far from exhausted.  The same principle applies at the
/// current 32 KiB initial size — the smaller the threshold, the less
/// reactive growth needed for ordinary deep call chains.
///
/// The reactive SIGBUS/SIGSEGV growth handler remains the safety net for the
/// rare case where a goroutine exhausts the remaining guard zone between two
/// scheduling points without ever calling `gosched`.
///
/// # Safety
/// Must be called from g0; `gp` must be about to be resumed via `gogo`.
pub(crate) unsafe fn grow_stack_if_needed(gp: *mut G) {
    let sp   = unsafe { (*gp).sched.sp };
    let lo   = unsafe { (*gp).stack.lo };

    // sp == 0 means the goroutine has never run yet (initial call).
    if sp == 0 || sp < lo + STACK_GUARD {
        // Growing here avoids the SIGSEGV race on the very first quantum.
        if sp != 0 {
            // Stack is nearly full; proactively double it.
            let old_stack = Stack {
                lo: unsafe { (*gp).stack.lo },
                hi: unsafe { (*gp).stack.hi },
            };
            let old_size = old_stack.hi - old_stack.lo;
            if old_size < STACK_MAX {
                let new_size  = (old_size * 2).min(STACK_MAX);
                // Proactive growth runs on g0 (non-signal), so it may use the
                // pooled allocator/free — reusing a cached stack of the target
                // size class and returning the old one for a future grow.
                let new_stack = unsafe {
                    stack_pool_alloc(new_size)
                        .expect("grow_stack_if_needed: allocation failed")
                };
                let delta = unsafe { copystack(gp, &old_stack, &new_stack) };
                unsafe {
                    (*gp).stack       = Stack { lo: new_stack.lo, hi: new_stack.hi };
                    (*gp).stackguard0 = new_stack.lo + STACK_GUARD;
                }
                unsafe { stack_pool_free(&old_stack) };
                let _ = delta;
            }
        }
    }
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn alloc_write_free() {
        unsafe {
            let stack = stack_alloc().expect("stack_alloc failed");
            let ps = page_size();

            assert_eq!(stack.hi - stack.lo, GOROUTINE_STACK_BYTES);
            // stack.lo is the first byte of the usable region, aligned to a
            // page boundary (it sits immediately above the guard page).
            assert_eq!(stack.lo % ps, 0);
            // stack.hi is stack.lo + GOROUTINE_STACK_BYTES.  On platforms with
            // large pages (e.g. macOS AArch64: 16 KiB), hi may not be aligned
            // when GOROUTINE_STACK_BYTES < page_size.  Do not assert alignment.
            assert!(stack.hi > stack.lo);

            let top = (stack.hi - 8) as *mut u64;
            top.write(0xDEAD_BEEF_CAFE_BABE);
            assert_eq!(top.read(), 0xDEAD_BEEF_CAFE_BABE);

            stack_free(&stack);
        }
    }

    #[test]
    fn page_size_sanity() {
        let ps = page_size();
        assert!(ps.is_power_of_two());
        assert!(ps >= 4096);
        println!("page_size = {ps}");
    }

    #[test]
    fn page_size_concurrent() {
        let handles: Vec<_> = (0..8)
            .map(|_| std::thread::spawn(page_size))
            .collect();
        let sizes: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect();
        assert!(sizes.windows(2).all(|w| w[0] == w[1]));
    }

    /// stack_alloc_size: variable-size allocation round-trips correctly.
    #[test]
    fn variable_size_alloc() {
        unsafe {
            for &size in &[8 * 1024usize, 16 * 1024, 32 * 1024, 64 * 1024] {
                let stack = stack_alloc_size(size).unwrap();
                assert_eq!(stack.hi - stack.lo, size);
                stack_free(&stack);
            }
        }
    }

    /// `pool_class` maps only in-range powers of two to a bucket.
    #[test]
    fn pool_class_bounds() {
        assert_eq!(pool_class(1 << POOL_MIN_CLASS), Some(0));
        assert_eq!(
            pool_class(1 << POOL_MAX_CLASS),
            Some((POOL_MAX_CLASS - POOL_MIN_CLASS) as usize)
        );
        // Below range, above range, and non-power-of-two are all rejected.
        assert_eq!(pool_class(1 << (POOL_MIN_CLASS - 1)), None);
        assert_eq!(pool_class(1 << (POOL_MAX_CLASS + 1)), None);
        assert_eq!(pool_class(48 * 1024), None);
    }

    /// The pooled alloc/free API round-trips: a stack obtained from
    /// `stack_pool_alloc` has the requested size, its usable region is
    /// writable, and it can be returned via `stack_pool_free` and re-acquired.
    ///
    /// Identity (whether the *same* mapping comes back) is deliberately NOT
    /// asserted: the pool is a process-global shared with the runtime's own
    /// concurrent tests, so a peer could pop the parked entry first.  Reuse is
    /// covered by the `many_goroutines` integration runs.
    #[test]
    fn pool_alloc_free_round_trip() {
        for &cls in &[POOL_MIN_CLASS, POOL_MAX_CLASS] {
            let size = 1usize << cls;
            unsafe {
                let s1 = stack_pool_alloc(size).unwrap();
                assert_eq!(s1.hi - s1.lo, size);
                let top = (s1.hi - 8) as *mut u64;
                top.write(0xDEAD_BEEF_F00D_BABE);
                assert_eq!(top.read(), 0xDEAD_BEEF_F00D_BABE);
                stack_pool_free(&s1);

                // A second alloc of the same class also yields a usable stack
                // (served from the pool or freshly mapped — both valid).
                let s2 = stack_pool_alloc(size).unwrap();
                assert_eq!(s2.hi - s2.lo, size);
                let top2 = (s2.hi - 8) as *mut u64;
                top2.write(0x0102_0304_0506_0708);
                assert_eq!(top2.read(), 0x0102_0304_0506_0708);
                stack_pool_free(&s2);
            }
        }
    }

    /// `stackpool_off` honours `GOLIB_STACKPOOL_OFF`; with the pool disabled,
    /// alloc/free still round-trip (straight through to mmap/munmap).
    #[test]
    fn pool_ablation_round_trips() {
        // Don't mutate the process-global env in a way that races other tests;
        // just exercise the disabled-path wrappers by size outside pool range,
        // which always falls through to raw alloc/free regardless of the lever.
        let size = 8 * 1024usize; // below POOL_MIN_CLASS → never pooled
        unsafe {
            let s = stack_pool_alloc(size).unwrap();
            assert_eq!(s.hi - s.lo, size);
            stack_pool_free(&s);
        }
    }
}