syd 3.58.0

//
// Syd: rock-solid application kernel
// src/kcov/abi.rs: KCOV ABI handlers
//
// Copyright (c) 2025, 2026 Ali Polatel <alip@chesswob.org>
// SPDX-License-Identifier: GPL-3.0

use std::{
    cell::Cell,
    fmt,
    num::NonZeroUsize,
    os::fd::RawFd,
    ptr::{self, NonNull},
    sync::{
        atomic::{AtomicU64, Ordering},
        Arc, OnceLock, RwLock,
    },
};

use libc::{c_long, c_void};
use libseccomp::{ScmpArch, ScmpNotifResp};
use memchr::arch::all::is_equal;
use nix::{
    errno::Errno,
    fcntl::SealFlag,
    sys::mman::{mmap, munmap, MapFlags, ProtFlags},
    unistd::Pid,
};
use serde::{Serialize, Serializer};

use crate::{
    config::{KCOV_HEART_BEAT, SAFE_MFD_FLAGS},
    cookie::{safe_ftruncate, safe_memfd_create},
    fd::{fd_dev_inode, seal_memfd},
    hash::SydHashMap,
    ioctl::{Ioctl, IoctlMap},
    kcov::{
        clear_kcov_handle, clear_tls_sink, get_kcov_tid, get_kcov_tid_local, get_kcov_tid_remote,
        get_tls_sink, set_kcov_tid, set_tls_sink, unbind_kcov_tid, Kcov, KcovId, TraceMode,
    },
    lookup::MaybeFd,
    proc::util::{proc_kcov_read_id, proc_tgid},
    req::UNotifyEventRequest,
};

// Set the per-thread syscall number in the C stub's TLS.
extern "C" {
    fn syd_kcov_set_syscall(nr: c_long);
}

// Per-thread syscall number (Rust mirror of the C stub's tls_sys).
thread_local! {
    static TLS_SYS: Cell<c_long> = const { Cell::new(-1) };
}

// Per-thread cached KCOV context.
thread_local! {
    static TLS_CTX: Cell<(usize, u64, u8, i32)> = const { Cell::new((0, 0, 8, -1)) };
}

// Per-thread reference keeping mapping cached in TLS_CTX alive.
thread_local! {
    static TLS_MAP: Cell<Option<Arc<KcovMapping>>> = const { Cell::new(None) };
}

// Thread-local flag to prevent recursion in coverage hooks.
thread_local! {
    static TLS_REC: Cell<bool> = const { Cell::new(false) };
}

pub(crate) const KCOV_TEXT_BASE: u64 = 0xFFFF_FFFF_8000_0000;
pub(crate) const KCOV_TEXT_MASK: u64 = 0x3FFF_FFF0;

// RAII guard to temporarily disable KCOV coverage collection for current thread.
pub(crate) struct KcovSnap {
    prev: bool,
}

impl KcovSnap {
    pub(crate) fn new() -> Self {
        let prev = TLS_REC.with(|c| {
            let p = c.get();
            c.set(true);
            p
        });
        Self { prev }
    }
}

impl Drop for KcovSnap {
    fn drop(&mut self) {
        TLS_REC.with(|c| c.set(self.prev));
    }
}

// Clear per-thread KCOV context cache.
fn tls_ctx_clear() {
    TLS_CTX.with(|c| c.set((0, 0, 8, -1)));
    TLS_MAP.with(|c| c.set(None));
}

// Set active syscall number for KCOV coverage diversity.
//
// Must be called before `kcov_enter_for` so that heartbeat PCs and
// sanitizer-hook PCs both carry the same syscall tag.
pub(crate) fn kcov_set_syscall(nr: c_long) {
    // Set Rust-side TLS for heartbeat mixing.
    TLS_SYS.with(|c| c.set(nr));

    // SAFETY: Set C-side TLS for sanitizer hook mixing.
    unsafe { syd_kcov_set_syscall(nr) };
}

// Mix syscall number into a PC.
//
// Must produce the same transform as the C version.
// See: src/kcov/stub.c
fn mix_syscall(mut pc: u64) -> u64 {
    let nr = TLS_SYS.with(|c| c.get());
    if nr > 0 {
        pc ^= (nr as u64).wrapping_mul(0x517c_c1b7_2722_0a95);
    }
    pc
}

// Map a stable site ID into a PC-shaped value.
//
// Must produce the same transform as the C version.
// See: src/kcov/stub.c
const fn to_canon_pc(v: u64) -> u64 {
    // Enforce 16-byte alignment.
    let v = v & !0xFu64;

    if cfg!(target_pointer_width = "64") {
        // Map into kernel text range that syzkaller expects.
        // No bit overlap between base and mask, so OR == addition.
        KCOV_TEXT_BASE | (v & KCOV_TEXT_MASK)
    } else {
        // 32-bit: put into high region, keep alignment.
        let x = (v as u32) & 0x0FFF_FFF0u32;
        (0x8000_0000u32 | x) as u64
    }
}

// Encode KCOV comparison type.
//
// type bit 0   : KCOV_CMP_CONST
// type bits 1-2: size code (1->0, 2->2, 4->4, 8->6)
fn kcov_cmp_type(size_bytes: u8, is_const: bool) -> u64 {
    let size_code = match size_bytes {
        1 => 0u64,
        2 => 2u64,
        4 => 4u64,
        8 => 6u64,
        _ => 6u64,
    };
    size_code | u64::from(is_const)
}

// Coverage-area word size, in bytes.
const KCOV_WORDSIZE: u8 = 8;

fn store_count(map: usize, val: u64) {
    // SAFETY: Word 0 is 8-byte aligned in mapped area.
    unsafe { (*(map as *const AtomicU64)).store(val, Ordering::Release) };
}

macro_rules! pc_append {
    ($ty:ty, $map:expr, $words:expr, $val:expr) => {{
        // SAFETY: word 0 and payload slots lie within mapped area.
        let count = unsafe { &*($map as *const $ty) };
        let mut cnt = count.load(Ordering::Acquire);
        loop {
            let pos = cnt.wrapping_add(1);
            if u64::from(pos) >= $words {
                return;
            }
            match count.compare_exchange_weak(cnt, pos, Ordering::AcqRel, Ordering::Acquire) {
                Ok(_) => {
                    // SAFETY: pos < words, slot within mapped payload.
                    let slot = unsafe { &*($map as *const $ty).add(pos as usize) };
                    slot.store($val, Ordering::Relaxed);
                    return;
                }
                Err(actual) => cnt = actual,
            }
        }
    }};
}

fn zero_map(map: usize, len: usize) {
    if map == 0 || len == 0 {
        return;
    }
    // SAFETY: `map` owns `len` writable bytes.
    unsafe { ptr::write_bytes(map as *mut u8, 0, len) };
}

fn live_update_pc_clamped(ctx: &KcovCtx, pc: u64) {
    let base = ctx.base();
    if ctx.mode != Some(TraceMode::Pc) || ctx.words <= 1 || base == 0 {
        return;
    }

    pc_append!(AtomicU64, base, ctx.words as u64, pc);
}

// Refcounted KCOV shared mapping.
pub(crate) struct KcovMapping {
    pub(crate) base: usize,
    pub(crate) len: usize,
}

impl Drop for KcovMapping {
    fn drop(&mut self) {
        if self.len == 0 {
            return;
        }
        if let Some(ptr) = NonNull::new(self.base as *mut c_void) {
            // SAFETY: base/len is our own MAP_SHARED mapping.
            let _ = unsafe { munmap(ptr, self.len) };
        }
    }
}

// Per-TID kcov context.
pub(crate) struct KcovCtx {
    pub(crate) tid: Pid,
    pub(crate) tgid: Pid,
    pub(crate) dev: u64,
    pub(crate) map: Option<Arc<KcovMapping>>,
    pub(crate) words: usize,
    pub(crate) wordsize: u8,
    pub(crate) mode: Option<TraceMode>,
}

impl KcovCtx {
    // Base address of shared mapping, 0 if unmapped.
    fn base(&self) -> usize {
        self.map.as_ref().map_or(0, |m| m.base)
    }
}

//
// Singletons
//

static KCOV_REG: OnceLock<RwLock<SydHashMap<KcovId, KcovCtx>>> = OnceLock::new();
pub(crate) fn kcov_reg() -> &'static RwLock<SydHashMap<KcovId, KcovCtx>> {
    KCOV_REG.get_or_init(|| RwLock::new(SydHashMap::default()))
}

static KCOV_MGR: OnceLock<Kcov> = OnceLock::new();
pub(crate) fn kcov_mgr() -> &'static Kcov {
    KCOV_MGR.get_or_init(Kcov::new)
}

pub(crate) fn kcov_check_mmap(dev: u64, inode: u64, offset: u64, length: u64) -> Result<(), Errno> {
    let (words, wordsize) = {
        let reg = kcov_reg().read().unwrap_or_else(|e| e.into_inner());
        match reg.get(&KcovId(inode)) {
            Some(ctx) if ctx.dev == dev && ctx.map.is_none() => return Err(Errno::EINVAL),
            Some(ctx) if ctx.dev == dev => (ctx.words, ctx.wordsize),
            _ => return Ok(()),
        }
    };

    let size = words as u64 * u64::from(wordsize);
    let page = *crate::config::PAGE_SIZE;

    let vma_len = match length.checked_add(page - 1) {
        Some(sum) => sum & !(page - 1),
        None => return Err(Errno::ENOMEM),
    };

    if offset != 0 || vma_len != size {
        return Err(Errno::EINVAL);
    }

    Ok(())
}

//
// Public API
//

// Create a named memfd for kcov, register per-TID context.
pub(crate) fn kcov_open(tid: Pid) -> Result<MaybeFd, Errno> {
    let memfd = safe_memfd_create(c"syd-kcov", *SAFE_MFD_FLAGS)?;
    let (dev, kcov_id) = fd_dev_inode(&memfd)?;
    let tgid = proc_tgid(tid).unwrap_or(tid);
    kcov_mgr().open(kcov_id, tid, tgid)?;

    {
        let kcov_id = KcovId(kcov_id);
        let _snap = KcovSnap::new(); // pause coverage during kcov_reg().
        let mut map = kcov_reg().write().unwrap_or_else(|e| e.into_inner());
        map.insert(
            kcov_id,
            KcovCtx {
                tid,
                tgid,
                dev,
                map: None,
                words: 0,
                wordsize: 8,
                mode: None,
            },
        );
    }

    // Hand the original memfd back to caller.
    // This fd will not be closed on drop.
    Ok(memfd.into())
}

// Argument for KCOV_REMOTE_ENABLE ioctl.
#[repr(C)]
#[derive(Debug, Default, Copy, Clone)]
struct KcovRemoteArg {
    trace_mode: u32,
    area_size: u32,
    num_handles: u32,
    _pad: u32,
    common_handle: u64,
}

const KCOV_SUBSYSTEM_MASK: u64 = 0xffu64 << 56;
const KCOV_INSTANCE_MASK: u64 = 0xffff_ffff;
const KCOV_SUBSYSTEM_COMMON: u64 = 0x00u64 << 56;
const KCOV_SUBSYSTEM_USB: u64 = 0x01u64 << 56;
const KCOV_REMOTE_MAX_HANDLES: u32 = 0x100;

fn kcov_check_handle(
    handle: u64,
    common_valid: bool,
    uncommon_valid: bool,
    zero_valid: bool,
) -> bool {
    if handle & !(KCOV_SUBSYSTEM_MASK | KCOV_INSTANCE_MASK) != 0 {
        return false;
    }
    match handle & KCOV_SUBSYSTEM_MASK {
        KCOV_SUBSYSTEM_COMMON => {
            if handle & KCOV_INSTANCE_MASK != 0 {
                common_valid
            } else {
                zero_valid
            }
        }
        KCOV_SUBSYSTEM_USB => uncommon_valid,
        _ => false,
    }
}

static KCOV_HANDLE_MAP: OnceLock<RwLock<SydHashMap<u64, KcovId>>> = OnceLock::new();
fn kcov_handle_map() -> &'static RwLock<SydHashMap<u64, KcovId>> {
    KCOV_HANDLE_MAP.get_or_init(|| RwLock::new(SydHashMap::default()))
}

pub(crate) fn kcov_handle_drop(id: KcovId) {
    let mut map = kcov_handle_map().write().unwrap_or_else(|e| e.into_inner());
    map.retain(|_, owner| *owner != id);
}

pub(crate) fn kcov_handle_lookup(handle: u64) -> Option<KcovId> {
    let map = kcov_handle_map().read().unwrap_or_else(|e| e.into_inner());
    map.get(&handle).copied()
}

pub(crate) fn kcov_clear_mode(id: KcovId) {
    let _snap = KcovSnap::new(); // pause coverage during kcov_reg().
    let mut map = kcov_reg().write().unwrap_or_else(|e| e.into_inner());
    if let Some(ctx) = map.get_mut(&id) {
        ctx.mode = None;
    }
}

pub(crate) fn kcov_remove_tid<F: Fn(KcovId, Pid) -> Option<Pid>>(tid: Pid, rebind: F) {
    tls_ctx_clear();
    let mut dead: Vec<KcovId> = Vec::new();
    {
        let _snap = KcovSnap::new(); // pause coverage during kcov_reg().
        let mut map = kcov_reg().write().unwrap_or_else(|e| e.into_inner());
        map.retain(|id, ctx| {
            if ctx.tid != tid {
                return true;
            }
            if let Some(new_tid) = rebind(*id, ctx.tgid) {
                ctx.tid = new_tid;
                true
            } else {
                dead.push(*id);
                false
            }
        });
    }
    for id in dead {
        kcov_handle_drop(id);
    }
}

pub(crate) fn is_kcov_ioctl(request: &UNotifyEventRequest) -> bool {
    #[expect(clippy::cast_possible_truncation)]
    let req = Ioctl::from(request.scmpreq.data.args[1] as u32);
    KcovIoctl::try_from((req, ScmpArch::native())).is_ok()
}

#[expect(clippy::cognitive_complexity)]
pub(crate) fn kcov_ioctl(request: &UNotifyEventRequest) -> Result<ScmpNotifResp, Errno> {
    let tid = request.scmpreq.pid();

    let fd = match RawFd::try_from(request.scmpreq.data.args[0]) {
        Ok(fd) if fd >= 0 => fd,
        _ => return Err(Errno::EBADF),
    };

    // Resolve the KcovId from memfd inode.
    let kcov_id = match proc_kcov_read_id(tid, fd) {
        Ok(id) => id,
        Err(_) => return Err(Errno::ENOTTY),
    };

    // Decode native ioctl(2) by NAME.
    // Linux truncates cmd to unsigned int.
    #[expect(clippy::cast_possible_truncation)]
    let kcov_req = Ioctl::from(request.scmpreq.data.args[1] as u32);
    let kcov_arg = request.scmpreq.data.args[2];
    let kcov_cmd = match KcovIoctl::try_from((kcov_req, ScmpArch::native())) {
        Ok(cmd) => cmd,
        Err(_) => return Err(Errno::ENOTTY),
    };

    #[expect(clippy::cast_possible_truncation)]
    let result = match kcov_cmd {
        KcovIoctl::InitTrace => {
            let words = kcov_arg;
            let wordsize = KCOV_WORDSIZE;

            kcov_mgr().init_trace(kcov_id, words, wordsize)?;

            let words = words as usize;
            let len = match words.checked_mul(usize::from(wordsize)) {
                Some(len) => len,
                None => {
                    kcov_mgr().reset(kcov_id);
                    return Err(Errno::EOVERFLOW);
                }
            };

            let mapped = (|| -> Result<NonNull<c_void>, Errno> {
                let memfd = request.get_fd(fd)?;
                safe_ftruncate(&memfd, i64::try_from(len).or(Err(Errno::EINVAL))?)?;
                let flags = SealFlag::F_SEAL_SEAL | SealFlag::F_SEAL_SHRINK | SealFlag::F_SEAL_GROW;
                seal_memfd(&memfd, flags)?;
                let nz = NonZeroUsize::new(len).ok_or(Errno::EINVAL)?;
                // SAFETY: memfd is sized to len. MAP_SHARED is required.
                unsafe {
                    mmap(
                        None,
                        nz,
                        ProtFlags::PROT_READ | ProtFlags::PROT_WRITE,
                        MapFlags::MAP_SHARED,
                        &memfd,
                        0,
                    )
                }
            })();
            let base = match mapped {
                Ok(base) => base,
                Err(err) => {
                    kcov_mgr().reset(kcov_id);
                    return Err(err);
                }
            };

            let _snap = KcovSnap::new(); // pause coverage during kcov_reg().
            let mut map = kcov_reg().write().unwrap_or_else(|e| e.into_inner());
            let ctx = match map.get_mut(&kcov_id) {
                Some(ctx) => ctx,
                None => {
                    // SAFETY: Base is our fresh mapping of len bytes.
                    let _ = unsafe { munmap(base, len) };
                    kcov_mgr().reset(kcov_id);
                    return Err(Errno::ENOTTY);
                }
            };

            ctx.words = words;
            ctx.wordsize = wordsize;
            ctx.map = Some(Arc::new(KcovMapping {
                base: base.as_ptr() as usize,
                len,
            }));
            zero_map(base.as_ptr() as usize, len);

            Ok(ok0(request))
        }

        KcovIoctl::Enable => {
            {
                let map = kcov_reg().read().unwrap_or_else(|e| e.into_inner());
                let ctx = map.get(&kcov_id).ok_or(Errno::ENOTTY)?;
                if ctx.map.is_none() {
                    return Err(Errno::EINVAL);
                }
            }
            kcov_mgr().can_enable(kcov_id)?;

            if get_kcov_tid_local(tid).is_some() {
                return Err(Errno::EBUSY);
            }

            let mode = match kcov_arg {
                0 => TraceMode::Pc,
                1 => TraceMode::Cmp,
                _ => return Err(Errno::EINVAL),
            };

            kcov_mgr().enable(kcov_id, mode)?;

            {
                let _snap = KcovSnap::new(); // pause coverage during kcov_reg().
                let mut map = kcov_reg().write().unwrap_or_else(|e| e.into_inner());
                let ctx = map.get_mut(&kcov_id).ok_or(Errno::ENOTTY)?;
                ctx.mode = Some(mode);
            }
            set_kcov_tid(tid, kcov_id, false, 0);

            Ok(ok0(request))
        }

        KcovIoctl::RemoteEnable => {
            let mut num_handles = 0u32;

            // SAFETY: num_handles is a plain u32.
            let nbuf =
                unsafe { std::slice::from_raw_parts_mut(&raw mut num_handles as *mut u8, 4) };

            let noff = kcov_arg
                .checked_add(std::mem::offset_of!(KcovRemoteArg, num_handles) as u64)
                .ok_or(Errno::EFAULT)?;

            if request.read_mem(nbuf, noff, 4)? != 4 {
                return Err(Errno::EFAULT);
            }

            if num_handles > KCOV_REMOTE_MAX_HANDLES {
                return Err(Errno::EINVAL);
            }

            let mut arg = KcovRemoteArg::default();

            // SAFETY: KcovRemoteArg is repr(C) and all-zero initialised.
            let buf = unsafe {
                std::slice::from_raw_parts_mut(&raw mut arg as *mut u8, size_of::<KcovRemoteArg>())
            };

            if request.read_mem(buf, kcov_arg, buf.len())? != buf.len() {
                return Err(Errno::EFAULT);
            }

            let num = num_handles as usize;
            let mut handles: Vec<u64> = Vec::new();
            handles.try_reserve(num).or(Err(Errno::ENOMEM))?;
            handles.resize(num, 0);
            if num > 0 {
                // SAFETY: Handles owns num contiguous u64 words.
                let hbuf = unsafe {
                    std::slice::from_raw_parts_mut(
                        handles.as_mut_ptr() as *mut u8,
                        num * size_of::<u64>(),
                    )
                };
                let off = kcov_arg
                    .checked_add(size_of::<KcovRemoteArg>() as u64)
                    .ok_or(Errno::EFAULT)?;
                if request.read_mem(hbuf, off, hbuf.len())? != hbuf.len() {
                    return Err(Errno::EFAULT);
                }
            }

            if arg.num_handles != num_handles {
                return Err(Errno::EINVAL);
            }

            let wordsize = {
                let map = kcov_reg().read().unwrap_or_else(|e| e.into_inner());
                let ctx = map.get(&kcov_id).ok_or(Errno::ENOTTY)?;
                if ctx.map.is_none() {
                    return Err(Errno::EINVAL);
                }
                ctx.wordsize
            };
            kcov_mgr().can_enable(kcov_id)?;

            if get_kcov_tid_remote(tid).is_some() {
                return Err(Errno::EBUSY);
            }

            let mode = match arg.trace_mode {
                0 => TraceMode::Pc,
                1 => TraceMode::Cmp,
                _ => return Err(Errno::EINVAL),
            };

            if u64::from(arg.area_size) > (i64::MAX as u64) / u64::from(wordsize) {
                return Err(Errno::EINVAL);
            }

            let mut added: Vec<u64> = Vec::new();
            added
                .try_reserve(num.saturating_add(1))
                .or(Err(Errno::ENOMEM))?;

            kcov_mgr().enable(kcov_id, mode)?;

            let failed = {
                let mut hmap = kcov_handle_map().write().unwrap_or_else(|e| e.into_inner());
                let all = handles
                    .iter()
                    .map(|&handle| (handle, false))
                    .chain((arg.common_handle != 0).then_some((arg.common_handle, true)));
                let mut failed = None;
                for (handle, is_common) in all {
                    let valid = if is_common {
                        kcov_check_handle(handle, true, false, false)
                    } else {
                        kcov_check_handle(handle, false, true, false)
                    };
                    if !valid {
                        failed = Some(Errno::EINVAL);
                    } else if hmap.contains_key(&handle) {
                        failed = Some(Errno::EEXIST);
                    }
                    if failed.is_some() {
                        for done in &added {
                            hmap.remove(done);
                        }
                        break;
                    }
                    hmap.insert(handle, kcov_id);
                    added.push(handle);
                }
                failed
            };
            if let Some(errno) = failed {
                let _ = kcov_mgr().disable(kcov_id);
                clear_kcov_handle(tid);
                return Err(errno);
            }

            {
                let _snap = KcovSnap::new(); // pause coverage during kcov_reg().
                let mut map = kcov_reg().write().unwrap_or_else(|e| e.into_inner());
                let ctx = map.get_mut(&kcov_id).ok_or(Errno::ENOTTY)?;
                ctx.mode = Some(mode);
            }
            set_kcov_tid(tid, kcov_id, true, arg.common_handle);

            Ok(ok0(request))
        }

        KcovIoctl::Disable => {
            if kcov_arg != 0 {
                return Err(Errno::EINVAL);
            }

            let is_local = get_kcov_tid_local(tid) == Some(kcov_id);
            let is_remote = get_kcov_tid_remote(tid) == Some(kcov_id);
            if !is_local && !is_remote {
                return Err(Errno::EINVAL);
            }

            kcov_mgr().disable(kcov_id)?;
            unbind_kcov_tid(tid, kcov_id);
            kcov_clear_mode(kcov_id);
            if is_remote {
                kcov_handle_drop(kcov_id);
            }

            Ok(ok0(request))
        }

        KcovIoctl::ResetTrace => {
            if kcov_arg != 0 {
                return Err(Errno::EINVAL);
            }

            let _snap = KcovSnap::new(); // pause coverage during kcov_reg().
            let reg = kcov_reg().read().unwrap_or_else(|e| e.into_inner());
            let ctx = reg.get(&kcov_id).ok_or(Errno::ENOTTY)?;

            if ctx.map.is_some() {
                store_count(ctx.base(), 0);
            }

            Ok(ok0(request))
        }
    };

    result
}

// Attach KCOV for a thread on syscall dispatch.
pub(crate) fn kcov_enter_for(tid: Pid) -> Result<(), Errno> {
    // Use get_kcov_tid(tid) as source of truth.
    let kcov_id = match get_kcov_tid(tid) {
        Some(id) => {
            set_tls_sink(id);
            id
        }
        None => {
            clear_tls_sink();
            tls_ctx_clear();
            return Ok(());
        }
    };

    let reg = kcov_reg().read().unwrap_or_else(|e| e.into_inner());
    let ctx = match reg.get(&kcov_id) {
        Some(ctx) => ctx,
        None => {
            tls_ctx_clear();
            return Ok(());
        }
    };
    if ctx.mode.is_none() || ctx.map.is_none() {
        tls_ctx_clear();
        return Ok(());
    }

    let cached_mode = match ctx.mode {
        Some(TraceMode::Pc) => 0,
        Some(TraceMode::Cmp) => 1,
        None => -1,
    };
    TLS_CTX.with(|c| c.set((ctx.base(), ctx.words as u64, ctx.wordsize, cached_mode)));
    TLS_MAP.with(|c| c.set(ctx.map.clone()));

    if ctx.words > 0 {
        let (map, words, wordsize, mode) = (ctx.base(), ctx.words, ctx.wordsize, ctx.mode.unwrap());
        drop(reg);
        emit_heartbeats(map, words, wordsize, mode);
    }

    Ok(())
}

fn emit_heartbeats(map: usize, words: usize, wordsize: u8, mode: TraceMode) {
    if map == 0 {
        return;
    }
    let pc = to_canon_pc(mix_syscall(KCOV_HEART_BEAT));
    match mode {
        TraceMode::Pc if words > 1 => {
            pc_append!(AtomicU64, map, words as u64, pc);
        }
        TraceMode::Cmp => {
            let slots = (words as u64).saturating_mul(u64::from(wordsize)) / 8;
            let cap = if slots > 1 { (slots - 1) / 4 } else { 0 };
            // SAFETY: Word 0 is 8-byte aligned in mapped area.
            let count = unsafe { &*(map as *const AtomicU64) };
            let mut cnt = count.load(Ordering::Acquire);
            loop {
                if cnt >= cap {
                    return;
                }
                match count.compare_exchange_weak(cnt, cnt + 1, Ordering::AcqRel, Ordering::Acquire)
                {
                    Ok(_) => {
                        let rec = 1 + (cnt as usize) * 4;
                        let area = map as *const AtomicU64;
                        // SAFETY: Reserved record is within mapped area.
                        unsafe {
                            (*area.add(rec)).store(kcov_cmp_type(8, false), Ordering::Relaxed);
                            (*area.add(rec + 1)).store(1, Ordering::Relaxed);
                            (*area.add(rec + 2)).store(0, Ordering::Relaxed);
                            (*area.add(rec + 3)).store(pc, Ordering::Relaxed);
                        }
                        return;
                    }
                    Err(actual) => cnt = actual,
                }
            }
        }
        _ => {}
    }
}

// RAII guard for syscall's coverage window.
pub(crate) struct KcovScope;

impl KcovScope {
    pub(crate) fn enter(pid: Pid, nr: c_long) -> Self {
        kcov_set_syscall(nr);
        let _ = kcov_enter_for(pid);
        crate::kcov_edge!();
        Self
    }
}

impl Drop for KcovScope {
    fn drop(&mut self) {
        crate::kcov_edge!();
        clear_tls_sink();
        tls_ctx_clear();
    }
}

fn ok0(req: &UNotifyEventRequest) -> ScmpNotifResp {
    ScmpNotifResp::new(req.scmpreq.id, 0, 0, 0)
}

//
// FFI exports for C stub to access KCOV context
//

#[repr(C)]
pub(crate) struct kcov_ctx {
    // Shared mapping base address
    pub(crate) base: usize,
    // Buffer size in words
    pub(crate) words: u64,
    // Coverage-area word width in bytes (always 8)
    pub(crate) wordsize: u8,
    // 0=PC, 1=CMP
    pub(crate) mode: i32,
}

/// Get KCOV context for given ID.
///
/// Called from sanitizer hooks which may fire while kcov_enter_for()
/// holds a read lock. Reads from TLS cache populated by kcov_enter_for().
/// No lock required.
#[no_mangle]
pub extern "C" fn syd_kcov_get_ctx(out_ctx: *mut kcov_ctx) -> bool {
    if out_ctx.is_null() {
        return false;
    }

    // Context is identified by TLS, not by KCOV id.
    let (base, words, wordsize, mode) = TLS_CTX.with(|c| c.get());
    if base == 0 || words == 0 {
        return false;
    }

    // SAFETY: FFI boundary, out_ctx is trusted.
    unsafe {
        (*out_ctx).base = base;
        (*out_ctx).words = words;
        (*out_ctx).wordsize = wordsize;
        (*out_ctx).mode = mode;
    }
    true
}

//
// Sanitizer Coverage Hooks
//

#[derive(Debug, Copy, Clone, Eq, PartialEq)]
enum KcovIoctl {
    InitTrace,
    ResetTrace,
    Enable,
    RemoteEnable,
    Disable,
}

impl TryFrom<(Ioctl, ScmpArch)> for KcovIoctl {
    type Error = Errno;

    // Convert the given ioctl(2) and arch into a `KcovIoctl`.
    fn try_from(value: (Ioctl, ScmpArch)) -> Result<Self, Errno> {
        let (val, arch) = value;
        let map = IoctlMap::new(None, true);
        let names = map.get_names(val, arch)?.ok_or(Errno::ENOTTY)?;
        for name in names {
            let name = name.as_bytes();
            if is_equal(name, b"KCOV_INIT_TRACE") {
                return Ok(Self::InitTrace);
            } else if is_equal(name, b"KCOV_RESET_TRACE") {
                return Ok(Self::ResetTrace);
            } else if is_equal(name, b"KCOV_ENABLE") {
                return Ok(Self::Enable);
            } else if is_equal(name, b"KCOV_REMOTE_ENABLE") {
                return Ok(Self::RemoteEnable);
            } else if is_equal(name, b"KCOV_DISABLE") {
                return Ok(Self::Disable);
            }
        }
        Err(Errno::ENOTTY)
    }
}

impl fmt::Display for KcovIoctl {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let name = match self {
            Self::InitTrace => "kcov_init_trace",
            Self::ResetTrace => "kcov_reset_trace",
            Self::Enable => "kcov_enable",
            Self::RemoteEnable => "kcov_remote_enable",
            Self::Disable => "kcov_disable",
        };
        write!(f, "{name}")
    }
}

impl Serialize for KcovIoctl {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        serializer.serialize_str(&self.to_string())
    }
}

// Recording entry points used by the instrumentation glue.
#[inline(never)]
pub(crate) fn record_pc(pc: u64) -> Result<(), Errno> {
    if TLS_REC.with(|c| {
        if c.get() {
            true
        } else {
            c.set(true);
            false
        }
    }) {
        return Ok(());
    }

    let pc = to_canon_pc(mix_syscall(pc));

    if let Some(id) = get_tls_sink() {
        let map = kcov_reg().read().unwrap_or_else(|e| e.into_inner());
        if let Some(ctx) = map.get(&id) {
            live_update_pc_clamped(ctx, pc);
        }
    }

    TLS_REC.with(|c| c.set(false));
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_to_canon_pc_alignment_1() {
        let pc = to_canon_pc(0x10);
        assert_eq!(pc & 0xF, 0, "must be 16-byte aligned");
    }

    #[test]
    fn test_to_canon_pc_alignment_2() {
        let pc = to_canon_pc(0x1F);
        assert_eq!(pc & 0xF, 0, "unaligned input must be rounded down");
    }

    #[test]
    fn test_to_canon_pc_zero_1() {
        let pc = to_canon_pc(0);
        assert_eq!(pc & 0xF, 0);
    }

    #[test]
    fn test_to_canon_pc_kernel_range_1() {
        // On 64-bit, result must be in the high kernel address range.
        if cfg!(target_pointer_width = "64") {
            let pc = to_canon_pc(0x100);
            assert!(pc >= 0xFFFF_FFFF_8000_0000u64);
        }
    }

    #[test]
    fn test_kcov_cmp_type_size1_const_1() {
        assert_eq!(kcov_cmp_type(1, true), 0u64 | 1u64);
    }

    #[test]
    fn test_kcov_cmp_type_size1_not_const_1() {
        assert_eq!(kcov_cmp_type(1, false), 0u64);
    }

    #[test]
    fn test_kcov_cmp_type_size2_const_1() {
        assert_eq!(kcov_cmp_type(2, true), 2u64 | 1u64);
    }

    #[test]
    fn test_kcov_cmp_type_size4_1() {
        assert_eq!(kcov_cmp_type(4, false), 4u64);
    }

    #[test]
    fn test_kcov_cmp_type_size8_1() {
        assert_eq!(kcov_cmp_type(8, false), 6u64);
    }

    #[test]
    fn test_kcov_cmp_type_unknown_size_1() {
        assert_eq!(kcov_cmp_type(16, false), 6u64);
    }

    #[test]
    fn test_kcov_cmp_type_const_bit_1() {
        let with_const = kcov_cmp_type(8, true);
        let without_const = kcov_cmp_type(8, false);
        assert_eq!(with_const, without_const | 1);
    }

    #[test]
    fn test_kcov_remote_arg_layout_1() {
        assert_eq!(std::mem::offset_of!(KcovRemoteArg, num_handles), 8);
        assert_eq!(std::mem::offset_of!(KcovRemoteArg, common_handle), 16);
        assert_eq!(size_of::<KcovRemoteArg>(), 24);
    }

    #[test]
    fn test_kcov_check_mmap_1() {
        let dev = 0xdead_beef_u64;
        let ino = 0x0f1e_2d3c_4b5a_6979_u64;
        let page = *crate::config::PAGE_SIZE;
        let wordsize = 8u8;
        let words = (page / u64::from(wordsize)) as usize;

        {
            let mut reg = kcov_reg().write().unwrap_or_else(|e| e.into_inner());
            reg.insert(
                KcovId(ino),
                KcovCtx {
                    tid: Pid::from_raw(0),
                    tgid: Pid::from_raw(0),
                    dev,
                    map: Some(Arc::new(KcovMapping {
                        base: 0x42, // fake
                        len: 0,     // never munmap
                    })),
                    words,
                    wordsize,
                    mode: None,
                },
            );
        }

        assert!(kcov_check_mmap(dev, ino, 0, page).is_ok());
        assert!(kcov_check_mmap(dev ^ 1, ino, 0, 1).is_ok());
        assert!(kcov_check_mmap(dev, ino ^ 0xffff, 0, 1).is_ok());
        assert_eq!(
            kcov_check_mmap(dev, ino, 0, page + page),
            Err(Errno::EINVAL)
        );
        assert_eq!(kcov_check_mmap(dev, ino, page, page), Err(Errno::EINVAL));

        {
            let mut reg = kcov_reg().write().unwrap_or_else(|e| e.into_inner());
            reg.insert(
                KcovId(ino),
                KcovCtx {
                    tid: Pid::from_raw(0),
                    tgid: Pid::from_raw(0),
                    dev,
                    map: None,
                    words,
                    wordsize,
                    mode: None,
                },
            );
        }

        assert_eq!(kcov_check_mmap(dev, ino, 0, page), Err(Errno::EINVAL));
        assert_eq!(
            kcov_check_mmap(dev, ino, 0, page + page),
            Err(Errno::EINVAL)
        );

        {
            let mut reg = kcov_reg().write().unwrap_or_else(|e| e.into_inner());
            reg.remove(&KcovId(ino));
        }
    }
}