rigless 0.2.1

// Copyright (c) 2026 Rigless Labs.
// SPDX-License-Identifier: MIT OR Apache-2.0
//! The protocol's framing, "Debugging with GDB" for GDB 18.1, section E.1: a debugger's connection,
//! its packets and their checksums, the acknowledgments, and the hex the packets carry.

use std::fmt::Write as _;
use std::io::{self, ErrorKind, Read, Write};
use std::net::TcpStream;

use super::Held;

/// The largest packet the server takes, which it tells GDB, in bytes.
pub(super) const PACKET_SIZE: usize = 0x1000;

/// GDB's interrupt, sent on its own outside a packet.
const INTERRUPT: u8 = 0x03;

/// What a debugger's bytes come and go over: its socket, or what a fuzzer sends.
pub(super) trait Carry: Read + Write + Send {
    /// Makes a read answer at once, with what's there or with `WouldBlock`, or wait for a byte.
    fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()>;
}

impl Carry for TcpStream {
    fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> {
        TcpStream::set_nonblocking(self, nonblocking)
    }
}

/// What the debugger on this connection said it supports, "Debugging with GDB" for GDB 18.1,
/// section E.5's `qSupported`, which starts empty on every connection.
#[derive(Clone, Copy, Debug, Default)]
#[expect(
    clippy::struct_excessive_bools,
    reason = "each is one feature or mode the debugger names on its own, and no two make a state"
)]
pub(super) struct Negotiated {
    /// Thread-ids name their process too, `p1.1`, and the end of the run names it.
    pub(super) multiprocess: bool,
    /// A stop at a `Z0` breakpoint says `swbreak`, section E.4.
    pub(super) swbreak: bool,
    /// A stop at a `Z1` breakpoint says `hwbreak`.
    pub(super) hwbreak: bool,
    /// Neither side sends or expects `+` and `-` any more, section E.12. Only `QStartNoAckMode`
    /// turns it on, and nothing turns it off again.
    pub(super) no_ack: bool,
    /// `target extended-remote`'s `!`, under which a detach keeps the connection, section 20.1.1.
    pub(super) extended: bool,
    /// Non-stop mode, `QNonStop:1`, "Debugging with GDB" for GDB 18.1, sections E.10 and E.11: GDB
    /// doesn't wait on the core, which answers `OK` to a resume and tells of each stop with a
    /// notification.
    pub(super) non_stop: bool,
}

/// One debugger's connection and the bytes it sent that aren't a whole packet yet.
#[expect(
    clippy::struct_excessive_bools,
    reason = "whether it has the core, has taken it, an interrupt waits, a packet is dropped, the far end closed and a stop is told are six things apart"
)]
pub(super) struct Connection {
    pub(super) stream: Box<dyn Carry>,
    pub(super) negotiated: Negotiated,
    /// The debugger has the core: from when it connects, or attaches or runs under
    /// `target extended-remote`, until it detaches or kills it.
    pub(super) attached: bool,
    /// It has taken the core, which it does with its first packet, so a port probe that sends none
    /// changes nothing.
    pub(super) taken: bool,
    /// How it found the core, held or running and with its halting debug on or off, until it sends
    /// a whole packet. One that closes before then was a port probe, and leaves the core so.
    pub(super) unspoken: Option<(Held, bool)>,
    received: Vec<u8>,
    /// The last packet sent, for GDB asking for it again.
    sent: Vec<u8>,
    /// GDB's interrupt came while the core was held, so the next continue or step stops before the
    /// core runs, "Debugging with GDB" for GDB 18.1, section E.9.
    pub(super) interrupt: bool,
    /// The rest of a packet dropped for its length is still to come.
    dropping: bool,
    /// The debugger closed its end. What it sent before that is answered all the same.
    closed: bool,
    /// In non-stop mode, GDB was told of a stop it hasn't taken with `vStopped` yet, GDB 18.1's
    /// section E.10.
    pub(super) telling: bool,
}

impl Connection {
    /// The debugger at the far end of `stream`, which found the core as `found` says and has it
    /// now.
    pub(super) fn new(stream: Box<dyn Carry>, found: (Held, bool)) -> Self {
        Self {
            stream,
            negotiated: Negotiated::default(),
            attached: true,
            taken: false,
            unspoken: Some(found),
            received: Vec::new(),
            sent: Vec::new(),
            interrupt: false,
            dropping: false,
            closed: false,
            telling: false,
        }
    }

    /// Sends `contents` as a packet.
    pub(super) fn send(&mut self, contents: &str) -> io::Result<()> {
        self.sent = frame(b'$', contents);
        self.stream.write_all(&self.sent)
    }

    /// Sends `contents` as a notification, "Debugging with GDB" for GDB 18.1, section E.10, which
    /// GDB doesn't acknowledge or ask for again.
    pub(super) fn notify(&mut self, contents: &str) -> io::Result<()> {
        self.stream.write_all(&frame(b'%', contents))
    }

    /// Whether GDB sent its interrupt while the core ran, without waiting for anything else: the
    /// byte outside a packet, where a packet's own bytes are its own. GDB sends nothing else then,
    /// so a little is read at a time, and what else comes in waits for the core to stop, no more
    /// than a packet's worth of it, so a client that floods the connection fills nothing.
    pub(super) fn interrupted(&mut self) -> io::Result<bool> {
        self.receive_waiting(64)?;
        let mut interrupted = false;
        let mut inside = false;
        let mut checksum = 0;
        self.received.retain(|&byte| {
            if inside {
                inside = byte != b'#';
                checksum = if inside { 0 } else { 2 };
            } else if checksum > 0 {
                checksum -= 1;
            } else if byte == INTERRUPT {
                interrupted = true;
                return false;
            } else {
                inside = byte == b'$';
            }
            true
        });
        let over = self.received.len().saturating_sub(PACKET_SIZE + 3);
        self.received.drain(..over);
        if self.closed && !interrupted {
            return Err(io::Error::from(ErrorKind::UnexpectedEof));
        }
        Ok(interrupted)
    }

    /// Every whole packet and interrupt that has come in, without waiting for more, each answered
    /// as [`Connection::next_packet`] answers it.
    pub(super) fn poll(&mut self) -> io::Result<Vec<Framed>> {
        self.receive_waiting(4096)?;
        let mut framed = Vec::new();
        while let Some(each) = take_packet(&mut self.received, &mut self.dropping) {
            framed.extend(self.acknowledge(each)?);
        }
        if self.closed && framed.is_empty() {
            return Err(io::Error::from(ErrorKind::UnexpectedEof));
        }
        Ok(framed)
    }

    /// At the end of the run: takes in what the debugger sends until it closes its end or goes
    /// quiet for the read timeout, at most a few packets' worth, so the close finds nothing unread.
    /// A close that did would reset the connection, under the debugger's acknowledgment of the
    /// last thing it heard.
    pub(super) fn hear_out(&mut self) {
        let mut bytes = [0; PACKET_SIZE];
        for _ in 0..4 {
            if !matches!(self.stream.read(&mut bytes), Ok(1..)) {
                return;
            }
        }
    }

    /// Whether the debugger has closed its end, taking in without waiting what it sent before
    /// that, which is answered all the same. A debugger still sending more than a few packets'
    /// worth isn't closing.
    pub(super) fn closed(&mut self) -> bool {
        let most = self.received.len() + 4 * PACKET_SIZE;
        while !self.closed && self.received.len() < most {
            let before = self.received.len();
            if self.receive_waiting(PACKET_SIZE).is_err() {
                self.closed = true;
            }
            if self.received.len() == before {
                break;
            }
        }
        self.closed
    }

    /// The next whole packet's contents, waiting for it, with an interrupt before it noted.
    pub(super) fn next_packet(&mut self) -> io::Result<String> {
        loop {
            if let Some(framed) = take_packet(&mut self.received, &mut self.dropping) {
                match self.acknowledge(framed)? {
                    Some(Framed::Packet(contents)) => return Ok(contents),
                    Some(Framed::Interrupt) => self.interrupt = true,
                    _ => {}
                }
                continue;
            }
            if self.closed {
                return Err(io::Error::from(ErrorKind::UnexpectedEof));
            }
            let mut bytes = [0; 4096];
            match self.stream.read(&mut bytes)? {
                0 => self.closed = true,
                read => self.received.extend_from_slice(&bytes[..read]),
            }
        }
    }

    /// Takes up to `most` bytes of what's come in, without waiting for more, and notes when the
    /// debugger has closed its end.
    fn receive_waiting(&mut self, most: usize) -> io::Result<()> {
        if self.closed {
            return Ok(());
        }
        self.stream.set_nonblocking(true)?;
        let mut bytes = vec![0; most];
        let read = match self.stream.read(&mut bytes) {
            Ok(0) => {
                self.closed = true;
                Ok(0)
            }
            Ok(read) => Ok(read),
            Err(error) if error.kind() == ErrorKind::WouldBlock => Ok(0),
            Err(error) => Err(error),
        };
        self.stream.set_nonblocking(false)?;
        self.received.extend_from_slice(&bytes[..read?]);
        Ok(())
    }

    /// Answers what came in as section E.1 asks, a packet with `+`, a corrupt one with `-`, and a
    /// `-` with the last packet again, unless the debugger turned that off. Hands back a packet or
    /// an interrupt.
    fn acknowledge(&mut self, framed: Framed) -> io::Result<Option<Framed>> {
        let acknowledged = !self.negotiated.no_ack;
        match framed {
            Framed::Packet(_) => {
                self.unspoken = None;
                if acknowledged {
                    self.stream.write_all(b"+")?;
                }
                Ok(Some(framed))
            }
            Framed::Interrupt => Ok(Some(framed)),
            Framed::Corrupt if acknowledged => self.stream.write_all(b"-").map(|()| None),
            Framed::Again if acknowledged => self.stream.write_all(&self.sent).map(|()| None),
            Framed::Corrupt | Framed::Again => Ok(None),
        }
    }
}

/// What the front of a connection's bytes holds.
#[derive(Debug, PartialEq, Eq)]
pub(super) enum Framed {
    /// A packet whose checksum held, and its contents.
    Packet(String),
    /// A packet whose checksum didn't, or that ran past [`PACKET_SIZE`] with no end.
    Corrupt,
    /// GDB asked for the last packet again.
    Again,
    /// GDB's interrupt.
    Interrupt,
}

/// Takes the first thing `received` holds off its front, or `None` until a whole one is there. An
/// acknowledgment and anything else outside a packet is passed over and dropped, and so is a
/// packet that runs past [`PACKET_SIZE`] with no end, so a connection's bytes never pile up. That
/// sets `dropping`, and the rest of the packet is passed over as it comes, through its checksum, so
/// a `-` or an interrupt inside it is read as neither.
fn take_packet(received: &mut Vec<u8>, dropping: &mut bool) -> Option<Framed> {
    if *dropping {
        let Some(end) = received.iter().position(|&byte| byte == b'#') else {
            received.clear();
            return None;
        };
        if received.len() < end + 3 {
            received.drain(..end);
            return None;
        }
        received.drain(..end + 3);
        *dropping = false;
    }
    let Some(start) = received
        .iter()
        .position(|&byte| matches!(byte, b'$' | b'-' | INTERRUPT))
    else {
        received.clear();
        return None;
    };
    received.drain(..start);
    match received[0] {
        b'-' => {
            received.remove(0);
            return Some(Framed::Again);
        }
        INTERRUPT => {
            received.remove(0);
            return Some(Framed::Interrupt);
        }
        _ => {}
    }
    let Some(end) = received.iter().position(|&byte| byte == b'#') else {
        if received.len() - 1 > PACKET_SIZE {
            received.clear();
            *dropping = true;
            return Some(Framed::Corrupt);
        }
        return None;
    };
    let checksum = received.get(end + 1..end + 3)?;
    let sent = std::str::from_utf8(checksum)
        .ok()
        .and_then(|text| u8::from_str_radix(text, 16).ok());
    let contents = &received[1..end];
    let framed = if sent == Some(sum(contents)) {
        Framed::Packet(contents.iter().map(|&byte| char::from(byte)).collect())
    } else {
        Framed::Corrupt
    };
    received.drain(..end + 3);
    Some(framed)
}

fn sum(bytes: &[u8]) -> u8 {
    bytes.iter().fold(0, |sum, &byte| sum.wrapping_add(byte))
}

/// `contents` as a packet, or with `%` a notification: `start`, the contents with `#`, `$`, `}` and
/// `*` escaped, `#` and the checksum.
fn frame(start: u8, contents: &str) -> Vec<u8> {
    let mut escaped = Vec::with_capacity(contents.len());
    for byte in contents.bytes() {
        if matches!(byte, b'#' | b'$' | b'}' | b'*') {
            escaped.extend([b'}', byte ^ 0x20]);
        } else {
            escaped.push(byte);
        }
    }
    let mut framed = vec![start];
    framed.extend_from_slice(&escaped);
    framed.extend(format!("#{:02x}", sum(&escaped)).bytes());
    framed
}

pub(super) fn hex(text: &str) -> Option<u64> {
    u64::from_str_radix(text, 16).ok()
}

/// `bytes` as GDB writes them, two hex digits each.
pub(super) fn to_hex(bytes: &[u8]) -> String {
    let mut text = String::with_capacity(bytes.len() * 2);
    for byte in bytes {
        write!(text, "{byte:02x}").expect("a String takes what's written to it");
    }
    text
}

pub(super) fn from_hex(text: &str) -> Option<Vec<u8>> {
    (0..text.len())
        .step_by(2)
        .map(|at| u8::from_str_radix(text.get(at..at + 2)?, 16).ok())
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A client's end that has `bytes` to read, and then waits.
    struct Waiting(std::collections::VecDeque<u8>);

    impl Read for Waiting {
        fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
            if self.0.is_empty() {
                return Err(io::Error::from(ErrorKind::WouldBlock));
            }
            self.0.read(bytes)
        }
    }

    impl Write for Waiting {
        fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
            Ok(bytes.len())
        }

        fn flush(&mut self) -> io::Result<()> {
            Ok(())
        }
    }

    impl Carry for Waiting {
        fn set_nonblocking(&self, _nonblocking: bool) -> io::Result<()> {
            Ok(())
        }
    }

    /// While the core runs, GDB's interrupt is the byte outside a packet, and the same byte inside
    /// one, an `X` write's say, isn't. However much a client sends then, no more than a packet's
    /// worth of it waits.
    #[test]
    fn a_running_core_hears_its_interrupt_and_keeps_a_packets_worth() {
        let packet = b"$X20000000,1:\x03#7a";
        let sent = |bytes: &[u8]| {
            Connection::new(
                Box::new(Waiting(bytes.iter().copied().collect())),
                (Held::Running, false),
            )
        };
        let mut connection = sent(packet);
        assert!(!connection.interrupted().unwrap(), "inside a packet");
        assert_eq!(connection.received, packet);
        let mut connection = sent(&[packet.as_slice(), &[INTERRUPT]].concat());
        assert!(connection.interrupted().unwrap());
        assert_eq!(connection.received, packet);
        let mut connection = sent(&vec![b'a'; 64 << 10]);
        for _ in 0..1024 {
            assert!(!connection.interrupted().unwrap());
        }
        assert!(connection.received.len() <= PACKET_SIZE + 3);
    }

    #[test]
    fn a_packet_is_framed_and_read_back() {
        let framed = frame(b'$', "m8000000,4");
        assert_eq!(framed, b"$m8000000,4#25");
        let mut received = b"+".to_vec();
        received.extend_from_slice(&framed);
        received.extend_from_slice(b"$g#67");
        assert_eq!(
            take_packet(&mut received, &mut false),
            Some(Framed::Packet("m8000000,4".to_owned()))
        );
        assert_eq!(
            take_packet(&mut received, &mut false),
            Some(Framed::Packet("g".to_owned()))
        );
        assert!(received.is_empty());
    }

    #[test]
    fn a_packet_waits_for_its_checksum_and_a_bad_one_is_corrupt() {
        let take = |bytes: &[u8]| take_packet(&mut bytes.to_vec(), &mut false);
        assert_eq!(take(b"$g#6"), None);
        assert_eq!(take(b"$g#00"), Some(Framed::Corrupt));
        assert_eq!(take(b"\x03"), Some(Framed::Interrupt));
        assert_eq!(take(b"-"), Some(Framed::Again));
    }

    /// Bytes outside a packet are dropped as they come, and so is a packet with no end once it runs
    /// past the most the server takes, which is corrupt, so a connection's bytes never pile up.
    #[test]
    fn nothing_piles_up_outside_a_packet_or_past_the_largest_one() {
        let mut dropping = false;
        let mut received = b"++noise".to_vec();
        assert_eq!(take_packet(&mut received, &mut dropping), None);
        assert!(received.is_empty());
        let mut received = vec![b'+', b'$'];
        received.resize(2 + PACKET_SIZE, b'a');
        assert_eq!(
            take_packet(&mut received, &mut dropping),
            None,
            "as long as one can be"
        );
        received.push(b'a');
        assert_eq!(
            take_packet(&mut received, &mut dropping),
            Some(Framed::Corrupt)
        );
        assert!(received.is_empty());
        assert!(dropping);
    }

    /// The rest of a packet dropped for its length is passed over through its checksum, however it
    /// comes, so a `-` or an interrupt inside it is neither, and the packet after it is read.
    #[test]
    fn the_rest_of_a_dropped_packet_is_passed_over_through_its_checksum() {
        let mut dropping = true;
        let mut received = b"a-a\x03a#0".to_vec();
        assert_eq!(
            take_packet(&mut received, &mut dropping),
            None,
            "its checksum is still to come"
        );
        received.extend_from_slice(b"0$g#67");
        assert_eq!(
            take_packet(&mut received, &mut dropping),
            Some(Framed::Packet("g".to_owned()))
        );
        assert!(received.is_empty());
        assert!(!dropping);
    }

    #[test]
    fn a_reply_escapes_what_the_protocol_reserves() {
        assert_eq!(frame(b'$', "a}b"), b"$a}]b#9d");
    }

    #[test]
    fn hex_reads_back_what_it_writes() {
        assert_eq!(to_hex(&[0x00, 0xAB, 0x10]), "00ab10");
        assert_eq!(from_hex("00ab10"), Some(vec![0x00, 0xAB, 0x10]));
        assert_eq!(from_hex("0g"), None);
    }
}