supercode_harness/tools/

convert.rs

//! BP-2: document conversion for the read and web tools — the "returns it
//! in a form the model can actually use" half of three catalog rows:
//!
//! * [`pdf_text`] — PDF → extracted text pages (catalog:27 "Read tool
//!   returns images/PDFs/ipynb as model-visible content"). A small
//!   in-crate extractor over the PDF content streams (`flate2` for the
//!   `/FlateDecode` filter every real-world writer uses), not a full PDF
//!   renderer: it recovers the text layer, and says so honestly when a
//!   document has none (scanned/image-only or encrypted).
//! * [`notebook_markdown`] — `.ipynb` → cells rendered WITH their outputs
//!   (same row).
//! * [`html_to_markdown`] — HTML → markdown (catalog:44 "Fetch a URL,
//!   convert to markdown, return to model").
//!
//! Same "small parser over a crate" precedent as `config::glob_match`,
//! `tools::url_host` and `builtins::base64_encode`.

use std::io::Read as _;

// ---- PDF ------------------------------------------------------------------

/// Whether `path` has a `.pdf` extension (case-insensitive).
pub fn is_pdf_path(path: &std::path::Path) -> bool {
    path.extension()
        .and_then(|e| e.to_str())
        .is_some_and(|e| e.eq_ignore_ascii_case("pdf"))
}

/// One extracted PDF page (content stream, in file order).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PdfPage {
    /// 1-based ordinal of the content stream this text came from.
    pub index: usize,
    /// The text recovered from that stream.
    pub text: String,
}

/// What [`pdf_text`] recovered from a PDF's bytes.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PdfDocument {
    /// `/Type /Page` objects counted in the file (0 when unparseable).
    pub page_count: usize,
    /// Whether the document declares an `/Encrypt` dictionary — the usual
    /// reason a structurally fine PDF yields no text.
    pub encrypted: bool,
    /// Pages whose content stream yielded any text, in file order.
    pub pages: Vec<PdfPage>,
}

/// Extract the text layer of a PDF.
///
/// Never fails: a document with no recoverable text comes back with an
/// empty `pages`, which the caller renders as an honest structured summary
/// rather than as silence.
pub fn pdf_text(bytes: &[u8]) -> PdfDocument {
    let page_count = count_occurrences(bytes, b"/Type /Page")
        + count_occurrences(bytes, b"/Type/Page")
        - count_occurrences(bytes, b"/Type /Pages")
        - count_occurrences(bytes, b"/Type/Pages");
    let encrypted = find(bytes, b"/Encrypt").is_some();
    let mut pages = Vec::new();
    for (index, stream) in content_streams(bytes).into_iter().enumerate() {
        let text = text_from_content_stream(&stream);
        if !text.trim().is_empty() {
            pages.push(PdfPage {
                index: index + 1,
                text,
            });
        }
    }
    PdfDocument {
        page_count,
        encrypted,
        pages,
    }
}

fn count_occurrences(haystack: &[u8], needle: &[u8]) -> usize {
    let mut n = 0;
    let mut from = 0;
    while let Some(at) = find(&haystack[from..], needle) {
        n += 1;
        from += at + needle.len();
    }
    n
}

fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
    if needle.is_empty() || haystack.len() < needle.len() {
        return None;
    }
    haystack
        .windows(needle.len())
        .position(|window| window == needle)
}

/// Every `stream`…`endstream` payload, inflated when the object's
/// dictionary declares `/FlateDecode`, skipping the ones that are plainly
/// not text (images, embedded fonts/files).
fn content_streams(bytes: &[u8]) -> Vec<Vec<u8>> {
    let mut out = Vec::new();
    let mut at = 0usize;
    while let Some(rel) = find(&bytes[at..], b"stream") {
        let start = at + rel;
        // `endstream`/`endobj` also contain "stream"; require a token start.
        let is_token_start = start == 0 || !bytes[start - 1].is_ascii_alphanumeric();
        let mut body = start + b"stream".len();
        if !is_token_start {
            at = start + b"stream".len();
            continue;
        }
        // Skip the EOL that must follow the `stream` keyword.
        if bytes.get(body) == Some(&b'\r') {
            body += 1;
        }
        if bytes.get(body) == Some(&b'\n') {
            body += 1;
        }
        let Some(end_rel) = find(&bytes[body..], b"endstream") else {
            break;
        };
        let end = body + end_rel;
        // The dictionary is whatever precedes the `stream` keyword back to
        // the object header — enough to read the filter and subtype.
        let dict_start = bytes[..start]
            .windows(3)
            .rposition(|w| w == b"obj")
            .map(|p| p + 3)
            .unwrap_or(0);
        let dict = &bytes[dict_start..start];
        let payload = &bytes[body..end];
        at = end + b"endstream".len();
        if contains(dict, b"/Image")
            || contains(dict, b"/DCTDecode")
            || contains(dict, b"/JPXDecode")
            || contains(dict, b"/FontFile")
            || contains(dict, b"/EmbeddedFile")
        {
            continue;
        }
        let decoded = if contains(dict, b"/FlateDecode") {
            match inflate(payload) {
                Some(d) => d,
                None => continue,
            }
        } else if contains(dict, b"/Filter") {
            // Some other filter (LZW, RunLength, ASCII85…) — not decoded.
            continue;
        } else {
            payload.to_vec()
        };
        out.push(decoded);
    }
    out
}

fn contains(haystack: &[u8], needle: &[u8]) -> bool {
    find(haystack, needle).is_some()
}

/// zlib-inflate `data`, tolerating the odd writer that omits the zlib
/// header (raw deflate).
fn inflate(data: &[u8]) -> Option<Vec<u8>> {
    let mut out = Vec::new();
    let mut zlib = flate2::read::ZlibDecoder::new(data);
    if zlib.read_to_end(&mut out).is_ok() && !out.is_empty() {
        return Some(out);
    }
    out.clear();
    let mut raw = flate2::read::DeflateDecoder::new(data);
    if raw.read_to_end(&mut out).is_ok() && !out.is_empty() {
        return Some(out);
    }
    None
}

/// Pull the shown strings out of a decoded content stream: `(...) Tj`,
/// `(...) '`/`"`, and the `[ (..) -250 (..) ] TJ` array form, with
/// `Td`/`TD`/`T*`/`ET` treated as line breaks.
fn text_from_content_stream(stream: &[u8]) -> String {
    let mut out = String::new();
    let mut pending: Vec<String> = Vec::new();
    let mut i = 0usize;
    while i < stream.len() {
        match stream[i] {
            b'(' => {
                let (s, next) = read_literal_string(stream, i);
                pending.push(s);
                i = next;
            }
            b'<' if stream.get(i + 1) != Some(&b'<') => {
                let (s, next) = read_hex_string(stream, i);
                pending.push(s);
                i = next;
            }
            b'T' => {
                let op = &stream[i..(i + 2).min(stream.len())];
                if op == b"Tj" || op == b"TJ" {
                    out.push_str(&pending.join(""));
                    pending.clear();
                    i += 2;
                } else if op == b"Td" || op == b"TD" || op == b"T*" {
                    out.push_str(&pending.join(""));
                    pending.clear();
                    if !out.ends_with('\n') {
                        out.push('\n');
                    }
                    i += 2;
                } else {
                    i += 1;
                }
            }
            b'\'' | b'"' => {
                out.push_str(&pending.join(""));
                pending.clear();
                if !out.ends_with('\n') {
                    out.push('\n');
                }
                i += 1;
            }
            b'E' if stream[i..].starts_with(b"ET") => {
                out.push_str(&pending.join(""));
                pending.clear();
                if !out.ends_with('\n') {
                    out.push('\n');
                }
                i += 2;
            }
            _ => i += 1,
        }
    }
    out.push_str(&pending.join(""));
    // Collapse the blank runs an operator-level extraction inevitably makes.
    let mut cleaned = String::with_capacity(out.len());
    let mut blank = 0;
    for line in out.lines() {
        let line = line.trim_end();
        if line.is_empty() {
            blank += 1;
            if blank > 1 {
                continue;
            }
        } else {
            blank = 0;
        }
        cleaned.push_str(line);
        cleaned.push('\n');
    }
    cleaned.trim_end().to_string()
}

/// A `(...)` literal string with PDF escapes, starting at `open`.
/// Returns the decoded text and the index just past the closing paren.
fn read_literal_string(stream: &[u8], open: usize) -> (String, usize) {
    let mut bytes: Vec<u8> = Vec::new();
    let mut depth = 1usize;
    let mut i = open + 1;
    while i < stream.len() {
        match stream[i] {
            b'\\' => {
                i += 1;
                let Some(&c) = stream.get(i) else { break };
                match c {
                    b'n' => bytes.push(b'\n'),
                    b'r' => bytes.push(b'\r'),
                    b't' => bytes.push(b'\t'),
                    b'b' => bytes.push(8),
                    b'f' => bytes.push(12),
                    b'\n' => {}
                    b'0'..=b'7' => {
                        let mut val = 0u32;
                        let mut digits = 0;
                        while digits < 3 {
                            match stream.get(i) {
                                Some(&d @ b'0'..=b'7') => {
                                    val = val * 8 + u32::from(d - b'0');
                                    i += 1;
                                    digits += 1;
                                }
                                _ => break,
                            }
                        }
                        i -= 1;
                        bytes.push(val as u8);
                    }
                    other => bytes.push(other),
                }
                i += 1;
            }
            b'(' => {
                depth += 1;
                bytes.push(b'(');
                i += 1;
            }
            b')' => {
                depth -= 1;
                i += 1;
                if depth == 0 {
                    break;
                }
                bytes.push(b')');
            }
            c => {
                bytes.push(c);
                i += 1;
            }
        }
    }
    (decode_pdf_bytes(&bytes), i)
}

/// A `<...>` hex string starting at `open`.
fn read_hex_string(stream: &[u8], open: usize) -> (String, usize) {
    let mut digits: Vec<u8> = Vec::new();
    let mut i = open + 1;
    while i < stream.len() && stream[i] != b'>' {
        if stream[i].is_ascii_hexdigit() {
            digits.push(stream[i]);
        }
        i += 1;
    }
    if digits.len() % 2 == 1 {
        digits.push(b'0');
    }
    let bytes: Vec<u8> = digits
        .chunks(2)
        .map(|pair| {
            let hi = (pair[0] as char).to_digit(16).unwrap_or(0) as u8;
            let lo = (pair[1] as char).to_digit(16).unwrap_or(0) as u8;
            (hi << 4) | lo
        })
        .collect();
    (decode_pdf_bytes(&bytes), i + 1)
}

/// PDFDocEncoding is Latin-1-compatible for the printable range; a
/// UTF-16BE string (the other encoding a writer may emit) announces itself
/// with a BOM or with a run of NUL high bytes.
fn decode_pdf_bytes(bytes: &[u8]) -> String {
    let utf16 = bytes.len() >= 2
        && (bytes[0] == 0xFE && bytes[1] == 0xFF
            || (bytes.len() % 2 == 0
                && bytes.chunks(2).filter(|c| c[0] == 0).count() * 2 > bytes.len()));
    if utf16 {
        let body = if bytes[0] == 0xFE && bytes[1] == 0xFF {
            &bytes[2..]
        } else {
            bytes
        };
        let units: Vec<u16> = body
            .chunks(2)
            .filter(|c| c.len() == 2)
            .map(|c| u16::from_be_bytes([c[0], c[1]]))
            .collect();
        return String::from_utf16_lossy(&units);
    }
    bytes.iter().map(|&b| b as char).collect()
}

/// Render a PDF for the model: extracted pages, or an honest structured
/// summary when the document carries no recoverable text layer.
pub fn pdf_markdown(name: &str, bytes: &[u8]) -> String {
    let doc = pdf_text(bytes);
    if doc.pages.is_empty() {
        let why = if doc.encrypted {
            "the document is encrypted"
        } else {
            "no text layer was found (a scanned/image-only PDF, or a filter this \
             extractor does not decode)"
        };
        return format!(
            "[read_file: PDF {name} — {} bytes, {} page objects; no text extracted: {why}. \
             The bytes themselves were not decoded as text.]",
            bytes.len(),
            doc.page_count
        );
    }
    let mut out = format!(
        "[read_file: PDF {name} — {} bytes, {} page objects, text extracted from {} content \
         stream(s) in file order]\n",
        bytes.len(),
        doc.page_count,
        doc.pages.len()
    );
    for page in &doc.pages {
        out.push_str(&format!("\n--- page {} ---\n{}\n", page.index, page.text));
    }
    out
}

// ---- Jupyter notebooks ----------------------------------------------------

/// Whether `path` has an `.ipynb` extension (case-insensitive).
pub fn is_notebook_path(path: &std::path::Path) -> bool {
    path.extension()
        .and_then(|e| e.to_str())
        .is_some_and(|e| e.eq_ignore_ascii_case(super::NOTEBOOK_EXTENSION))
}

/// A notebook `source`/`text` field: either a string or an array of lines.
fn json_text(value: Option<&serde_json::Value>) -> String {
    match value {
        Some(serde_json::Value::String(s)) => s.clone(),
        Some(serde_json::Value::Array(items)) => items
            .iter()
            .filter_map(|i| i.as_str())
            .collect::<Vec<_>>()
            .join(""),
        _ => String::new(),
    }
}

/// Render a `.ipynb` for the model: every cell with its type, execution
/// count and its OUTPUTS (stdout/stderr streams, text results, errors) —
/// the half a raw JSON decode buries.
pub fn notebook_markdown(name: &str, bytes: &[u8]) -> String {
    let Ok(nb) = serde_json::from_slice::<serde_json::Value>(bytes) else {
        return format!(
            "[read_file: {name} is not valid Jupyter notebook JSON; {} bytes not decoded]",
            bytes.len()
        );
    };
    let cells = nb.get("cells").and_then(|c| c.as_array());
    let kernel = nb
        .get("metadata")
        .and_then(|m| m.get("kernelspec"))
        .and_then(|k| k.get("display_name").or_else(|| k.get("name")))
        .and_then(|n| n.as_str())
        .unwrap_or("unknown");
    let Some(cells) = cells else {
        return format!("[read_file: {name} has no `cells` array (kernel: {kernel})]");
    };
    let mut out = format!(
        "[read_file: Jupyter notebook {name} — {} cells, kernel {kernel}]\n",
        cells.len()
    );
    for (index, cell) in cells.iter().enumerate() {
        let kind = cell
            .get("cell_type")
            .and_then(|t| t.as_str())
            .unwrap_or("unknown");
        let exec = cell
            .get("execution_count")
            .and_then(|c| c.as_u64())
            .map(|c| format!(" [{c}]"))
            .unwrap_or_default();
        let source = json_text(cell.get("source"));
        out.push_str(&format!(
            "\n--- cell {index} ({kind}){exec} ---\n{source}\n"
        ));
        let Some(outputs) = cell.get("outputs").and_then(|o| o.as_array()) else {
            continue;
        };
        for output in outputs {
            let rendered = match output.get("output_type").and_then(|t| t.as_str()) {
                Some("stream") => {
                    let stream = output
                        .get("name")
                        .and_then(|n| n.as_str())
                        .unwrap_or("stdout");
                    format!("[{stream}]\n{}", json_text(output.get("text")))
                }
                Some("error") => {
                    let ename = output
                        .get("ename")
                        .and_then(|e| e.as_str())
                        .unwrap_or("Error");
                    let evalue = output.get("evalue").and_then(|e| e.as_str()).unwrap_or("");
                    let traceback = output
                        .get("traceback")
                        .and_then(|t| t.as_array())
                        .map(|lines| {
                            lines
                                .iter()
                                .filter_map(|l| l.as_str())
                                .collect::<Vec<_>>()
                                .join("\n")
                        })
                        .unwrap_or_default();
                    format!("[error] {ename}: {evalue}\n{traceback}")
                }
                Some(kind @ ("execute_result" | "display_data")) => {
                    let data = output.get("data");
                    let text = data
                        .and_then(|d| d.get("text/plain"))
                        .map(|t| json_text(Some(t)))
                        .unwrap_or_default();
                    let mime_note = data
                        .and_then(|d| d.as_object())
                        .map(|o| {
                            o.keys()
                                .filter(|k| k.as_str() != "text/plain")
                                .cloned()
                                .collect::<Vec<_>>()
                        })
                        .filter(|extra| !extra.is_empty())
                        .map(|extra| format!(" (also: {})", extra.join(", ")))
                        .unwrap_or_default();
                    format!("[{kind}{mime_note}]\n{text}")
                }
                other => format!("[{}]", other.unwrap_or("output")),
            };
            out.push_str(&format!("--- output ---\n{}\n", rendered.trim_end()));
        }
    }
    out
}

// ---- HTML -----------------------------------------------------------------

/// Convert an HTML document to markdown: headings, links, list items,
/// emphasis, code and block structure survive; `<script>`/`<style>`/
/// comments and every other tag are dropped, entities are decoded.
///
/// A small tag-walking converter, not a DOM: the model needs the readable
/// text and its structure, and a fetched page's markup is never trusted
/// input for anything but text.
pub fn html_to_markdown(html: &str) -> String {
    let bytes = html.as_bytes();
    let mut out = String::with_capacity(html.len() / 2);
    let mut i = 0usize;
    // Open-anchor href, waiting for its text.
    let mut link_href: Option<String> = None;
    let mut link_text = String::new();
    while i < bytes.len() {
        if bytes[i] == b'<' {
            if html[i..].starts_with("<!--") {
                i = html[i..]
                    .find("-->")
                    .map(|p| i + p + 3)
                    .unwrap_or(bytes.len());
                continue;
            }
            let Some(close) = html[i..].find('>') else {
                break;
            };
            let raw = &html[i + 1..i + close];
            let end = i + close + 1;
            let name = tag_name(raw);
            match name.as_str() {
                "script" | "style" | "noscript" | "svg" | "head" => {
                    let closing = format!("</{name}");
                    i = html[end..]
                        .find(&closing)
                        .map(|p| {
                            let from = end + p;
                            html[from..].find('>').map(|q| from + q + 1).unwrap_or(end)
                        })
                        .unwrap_or(bytes.len());
                    continue;
                }
                "br" => push_line(&mut out),
                "p" | "div" | "section" | "article" | "tr" | "table" | "blockquote" | "pre"
                | "ul" | "ol" => push_block(&mut out),
                "/p" | "/div" | "/section" | "/article" | "/tr" | "/table" | "/blockquote"
                | "/pre" | "/ul" | "/ol" => push_block(&mut out),
                "li" => {
                    push_line(&mut out);
                    out.push_str("- ");
                }
                "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
                    push_block(&mut out);
                    let level: usize = name[1..].parse().unwrap_or(1);
                    out.push_str(&"#".repeat(level));
                    out.push(' ');
                }
                "/h1" | "/h2" | "/h3" | "/h4" | "/h5" | "/h6" => push_block(&mut out),
                "code" | "/code" => out.push('`'),
                "strong" | "/strong" | "b" | "/b" => out.push_str("**"),
                "em" | "/em" | "i" | "/i" => out.push('*'),
                "a" => {
                    link_href = attribute(raw, "href");
                    link_text.clear();
                }
                "/a" => {
                    let text = link_text.trim().to_string();
                    match link_href.take() {
                        Some(href) if !text.is_empty() => {
                            out.push_str(&format!("[{text}]({href})"))
                        }
                        _ => out.push_str(&text),
                    }
                    link_text.clear();
                }
                "td" | "th" => out.push_str(" | "),
                _ => {}
            }
            i = end;
            continue;
        }
        let next = html[i..].find('<').map(|p| i + p).unwrap_or(bytes.len());
        let text = decode_entities(&html[i..next]);
        let mut collapsed = collapse_whitespace(&text);
        let target = if link_href.is_some() {
            &mut link_text
        } else {
            &mut out
        };
        if target.is_empty() || target.ends_with(char::is_whitespace) {
            collapsed = collapsed.trim_start().to_string();
        }
        target.push_str(&collapsed);
        i = next;
    }
    // Collapse the blank runs block-level tags leave behind.
    let mut cleaned = String::with_capacity(out.len());
    let mut blank = 0;
    for line in out.lines() {
        let line = line.trim_end();
        if line.is_empty() {
            blank += 1;
            if blank > 1 {
                continue;
            }
        } else {
            blank = 0;
        }
        cleaned.push_str(line);
        cleaned.push('\n');
    }
    cleaned.trim().to_string()
}

fn push_line(out: &mut String) {
    if !out.is_empty() && !out.ends_with('\n') {
        out.push('\n');
    }
}

fn push_block(out: &mut String) {
    if out.is_empty() {
        return;
    }
    while out.ends_with(' ') {
        out.pop();
    }
    if !out.ends_with("\n\n") {
        if out.ends_with('\n') {
            out.push('\n');
        } else {
            out.push_str("\n\n");
        }
    }
}

/// The lowercase tag name of a raw tag body (`/` kept for closing tags).
fn tag_name(raw: &str) -> String {
    let raw = raw.trim();
    let mut name = String::new();
    for (index, c) in raw.char_indices() {
        if index == 0 && c == '/' {
            name.push('/');
            continue;
        }
        if c.is_ascii_alphanumeric() {
            name.push(c.to_ascii_lowercase());
        } else {
            break;
        }
    }
    name
}

/// One attribute's value out of a raw tag body, single- or double-quoted.
fn attribute(raw: &str, name: &str) -> Option<String> {
    let lower = raw.to_ascii_lowercase();
    let mut from = 0usize;
    while let Some(at) = lower[from..].find(name) {
        let start = from + at;
        let before_ok = start == 0
            || !lower.as_bytes()[start - 1].is_ascii_alphanumeric()
                && lower.as_bytes()[start - 1] != b'-';
        let rest = &raw[start + name.len()..];
        let trimmed = rest.trim_start();
        if before_ok && trimmed.starts_with('=') {
            let value = trimmed[1..].trim_start();
            let quote = value.chars().next()?;
            if quote == '"' || quote == '\'' {
                let end = value[1..].find(quote)? + 1;
                return Some(decode_entities(&value[1..end]));
            }
            let end = value
                .find(|c: char| c.is_whitespace())
                .unwrap_or(value.len());
            return Some(decode_entities(&value[..end]));
        }
        from = start + name.len();
    }
    None
}

/// Collapse every whitespace run in a text node to a single space,
/// KEEPING the leading/trailing one: inter-tag spacing (`Hello
/// <strong>world</strong> and`) lives in exactly those runs, and the
/// caller drops a leading space when the output already ends in one.
fn collapse_whitespace(text: &str) -> String {
    let mut out = String::with_capacity(text.len());
    let mut space = false;
    for c in text.chars() {
        if c.is_whitespace() {
            space = true;
            continue;
        }
        if space {
            out.push(' ');
        }
        space = false;
        out.push(c);
    }
    if space {
        out.push(' ');
    }
    out
}

/// Decode the named entities a text-extraction path actually meets, plus
/// numeric (`&#123;` / `&#x7b;`) references.
pub fn decode_entities(text: &str) -> String {
    if !text.contains('&') {
        return text.to_string();
    }
    let mut out = String::with_capacity(text.len());
    let bytes = text.as_bytes();
    let mut i = 0usize;
    while i < bytes.len() {
        if bytes[i] != b'&' {
            let next = text[i..].find('&').map(|p| i + p).unwrap_or(bytes.len());
            out.push_str(&text[i..next]);
            i = next;
            continue;
        }
        let Some(semi) = text[i..].find(';').filter(|p| *p <= 10) else {
            out.push('&');
            i += 1;
            continue;
        };
        let entity = &text[i + 1..i + semi];
        let decoded = match entity {
            "amp" => Some('&'),
            "lt" => Some('<'),
            "gt" => Some('>'),
            "quot" => Some('"'),
            "apos" | "#39" => Some('\''),
            "nbsp" => Some(' '),
            "hellip" => Some('…'),
            "mdash" => Some('—'),
            "ndash" => Some('–'),
            "rsquo" => Some('’'),
            "lsquo" => Some('‘'),
            "ldquo" => Some('“'),
            "rdquo" => Some('”'),
            other => other
                .strip_prefix('#')
                .and_then(|n| match n.strip_prefix(['x', 'X']) {
                    Some(hex) => u32::from_str_radix(hex, 16).ok(),
                    None => n.parse::<u32>().ok(),
                })
                .and_then(char::from_u32),
        };
        match decoded {
            Some(c) => {
                out.push(c);
                i += semi + 1;
            }
            None => {
                out.push('&');
                i += 1;
            }
        }
    }
    out
}

// ---- search-result extraction --------------------------------------------

/// One result from an HTML search-results page.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SearchResult {
    /// The result's link text.
    pub title: String,
    /// The destination URL, unwrapped from the engine's redirector.
    pub url: String,
    /// The engine's snippet, when the page carries one.
    pub snippet: String,
}

/// BP-2 (catalog:45 "Provider/server-backed search"): pull results out of a
/// DuckDuckGo-style HTML results page — the `result__a` anchors and their
/// `result__snippet` siblings, in page order.
///
/// Structure-tolerant on purpose: an engine that changes its markup yields
/// zero results here, and the caller then falls back to the page as
/// markdown rather than pretending there were no hits.
pub fn parse_html_search_results(html: &str) -> Vec<SearchResult> {
    let titles = elements_with_class(html, "a", "result__a");
    let snippets = elements_with_class(html, "a", "result__snippet");
    let snippets = if snippets.is_empty() {
        elements_with_class(html, "div", "result__snippet")
    } else {
        snippets
    };
    let mut out = Vec::new();
    for (index, (attrs, inner)) in titles.into_iter().enumerate() {
        let Some(href) = attribute(&attrs, "href") else {
            continue;
        };
        let url = unwrap_redirector(&href);
        let title = html_to_markdown(&inner);
        if title.is_empty() || url.is_empty() {
            continue;
        }
        let snippet = snippets
            .get(index)
            .map(|(_, text)| html_to_markdown(text))
            .unwrap_or_default();
        out.push(SearchResult {
            title,
            url,
            snippet,
        });
    }
    out
}

/// Every `<tag …class="… wanted …">inner</tag>` in `html`, as
/// `(raw attributes, inner html)`. Nesting of the SAME tag inside a match
/// is not handled — search-result markup does not nest anchors.
fn elements_with_class(html: &str, tag: &str, wanted: &str) -> Vec<(String, String)> {
    let open = format!("<{tag}");
    let close = format!("</{tag}");
    let mut out = Vec::new();
    let mut from = 0usize;
    while let Some(at) = html[from..].find(&open) {
        let start = from + at;
        let Some(gt) = html[start..].find('>') else {
            break;
        };
        let attrs = &html[start + open.len()..start + gt];
        let body_start = start + gt + 1;
        from = body_start;
        let has_class = attribute(attrs, "class")
            .is_some_and(|class| class.split_whitespace().any(|c| c == wanted));
        if !has_class {
            continue;
        }
        let Some(end) = html[body_start..].find(&close) else {
            continue;
        };
        out.push((
            attrs.to_string(),
            html[body_start..body_start + end].to_string(),
        ));
    }
    out
}

/// A results page links through a redirector (`//duckduckgo.com/l/?uddg=…`);
/// the destination the model needs is the encoded parameter.
fn unwrap_redirector(href: &str) -> String {
    for key in ["uddg=", "url=", "u=", "q="] {
        if let Some(at) = href.find(key) {
            let value = &href[at + key.len()..];
            let end = value.find('&').unwrap_or(value.len());
            let decoded = percent_decode(&value[..end]);
            if decoded.starts_with("http") {
                return decoded;
            }
        }
    }
    if let Some(rest) = href.strip_prefix("//") {
        return format!("https://{rest}");
    }
    href.to_string()
}

/// Percent-decode a URL component (`%20`, `+` as space).
pub fn percent_decode(text: &str) -> String {
    let bytes = text.as_bytes();
    let mut out: Vec<u8> = Vec::with_capacity(bytes.len());
    let mut i = 0usize;
    while i < bytes.len() {
        match bytes[i] {
            b'%' if i + 2 < bytes.len() => {
                let hi = (bytes[i + 1] as char).to_digit(16);
                let lo = (bytes[i + 2] as char).to_digit(16);
                match (hi, lo) {
                    (Some(hi), Some(lo)) => {
                        out.push(((hi << 4) | lo) as u8);
                        i += 3;
                    }
                    _ => {
                        out.push(b'%');
                        i += 1;
                    }
                }
            }
            b'+' => {
                out.push(b' ');
                i += 1;
            }
            c => {
                out.push(c);
                i += 1;
            }
        }
    }
    String::from_utf8_lossy(&out).into_owned()
}

/// Render extracted results for the model: a numbered list of
/// `title — url` with each engine snippet under it.
pub fn render_search_results(query: &str, results: &[SearchResult]) -> String {
    let mut out = format!("[web_search: {} results for {query:?}]\n", results.len());
    for (index, result) in results.iter().enumerate() {
        out.push_str(&format!(
            "\n{}. {} — {}\n",
            index + 1,
            result.title,
            result.url
        ));
        if !result.snippet.is_empty() {
            out.push_str(&format!("   {}\n", result.snippet));
        }
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn html_converts_headings_links_and_lists_and_drops_scripts() {
        let html = "<html><head><title>t</title></head><body>\
            <script>var x = '<b>no</b>';</script>\
            <h1>Title</h1><p>Hello <strong>world</strong> &amp; friends.</p>\
            <ul><li>one</li><li><a href=\"https://example.com/a\">two</a></li></ul>\
            </body></html>";
        let md = html_to_markdown(html);
        assert!(md.contains("# Title"), "{md}");
        assert!(md.contains("Hello **world** & friends."), "{md}");
        assert!(md.contains("- one"), "{md}");
        assert!(md.contains("[two](https://example.com/a)"), "{md}");
        assert!(!md.contains("var x"), "script body leaked: {md}");
        assert!(!md.contains('<'), "raw markup leaked: {md}");
    }

    #[test]
    fn notebook_renders_cells_with_their_outputs() {
        let nb = serde_json::json!({
            "metadata": {"kernelspec": {"display_name": "Python 3"}},
            "cells": [
                {"cell_type": "markdown", "source": ["# Demo\n"]},
                {"cell_type": "code", "execution_count": 1,
                 "source": ["print('hi')\n", "1 + 1\n"],
                 "outputs": [
                    {"output_type": "stream", "name": "stdout", "text": ["hi\n"]},
                    {"output_type": "execute_result", "data": {"text/plain": ["2"]}}
                 ]},
                {"cell_type": "code", "source": ["boom()"],
                 "outputs": [{"output_type": "error", "ename": "NameError",
                              "evalue": "name 'boom' is not defined", "traceback": ["line 1"]}]}
            ]
        });
        let out = notebook_markdown("demo.ipynb", nb.to_string().as_bytes());
        assert!(out.contains("3 cells, kernel Python 3"), "{out}");
        assert!(out.contains("--- cell 0 (markdown) ---"), "{out}");
        assert!(out.contains("--- cell 1 (code) [1] ---"), "{out}");
        assert!(out.contains("print('hi')"), "{out}");
        assert!(out.contains("[stdout]\nhi"), "{out}");
        assert!(out.contains("[execute_result]\n2"), "{out}");
        assert!(
            out.contains("[error] NameError: name 'boom' is not defined"),
            "{out}"
        );
    }

    #[test]
    fn pdf_extracts_text_from_an_uncompressed_content_stream() {
        // A minimal one-page PDF with a plain (unfiltered) content stream.
        let pdf = b"%PDF-1.4\n1 0 obj\n<< /Type /Page >>\nendobj\n\
            2 0 obj\n<< /Length 60 >>\nstream\n\
            BT /F1 12 Tf 72 720 Td (Hello parity) Tj T* (second line) Tj ET\n\
            endstream\nendobj\ntrailer\n%%EOF\n";
        let doc = pdf_text(pdf);
        assert_eq!(doc.page_count, 1, "{doc:?}");
        assert!(!doc.encrypted);
        assert_eq!(doc.pages.len(), 1, "{doc:?}");
        assert!(doc.pages[0].text.contains("Hello parity"), "{doc:?}");
        assert!(doc.pages[0].text.contains("second line"), "{doc:?}");
        let md = pdf_markdown("x.pdf", pdf);
        assert!(md.contains("--- page 1 ---"), "{md}");
    }

    #[test]
    fn pdf_extracts_text_from_a_flate_compressed_content_stream() {
        use std::io::Write as _;
        let content = b"BT (compressed text) Tj ET";
        let mut encoder =
            flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
        encoder.write_all(content).unwrap();
        let compressed = encoder.finish().unwrap();
        let mut pdf = b"%PDF-1.7\n1 0 obj\n<< /Type /Page >>\nendobj\n\
            2 0 obj\n<< /Filter /FlateDecode >>\nstream\n"
            .to_vec();
        pdf.extend_from_slice(&compressed);
        pdf.extend_from_slice(b"\nendstream\nendobj\n%%EOF\n");
        let doc = pdf_text(&pdf);
        assert_eq!(doc.pages.len(), 1, "{doc:?}");
        assert!(doc.pages[0].text.contains("compressed text"), "{doc:?}");
    }

    #[test]
    fn pdf_with_no_text_layer_says_so_honestly() {
        let pdf = b"%PDF-1.4\n1 0 obj\n<< /Type /Page >>\nendobj\n%%EOF\n";
        let md = pdf_markdown("scan.pdf", pdf);
        assert!(md.contains("no text extracted"), "{md}");
        assert!(md.contains("1 page objects"), "{md}");
    }

    #[test]
    fn pdf_hex_and_utf16_strings_decode() {
        let pdf = b"%PDF-1.4\n1 0 obj\n<< /Type /Page >>\nendobj\n2 0 obj\n<< >>\nstream\n\
            BT <48656C6C6F> Tj ET\nendstream\nendobj\n%%EOF\n";
        let doc = pdf_text(pdf);
        assert!(doc.pages[0].text.contains("Hello"), "{doc:?}");
    }

    #[test]
    fn search_results_parse_titles_urls_and_snippets_out_of_a_results_page() {
        let html = r#"<html><body>
            <div class="result results_links">
              <a rel="nofollow" class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fdoc.rust-lang.org%2Fbook%2F&amp;rut=xyz">The Rust <b>Book</b></a>
              <a class="result__snippet" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fdoc.rust-lang.org%2Fbook%2F">The official book about the Rust language.</a>
            </div>
            <div class="result results_links">
              <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fcrates.io%2F">crates.io</a>
              <a class="result__snippet">The Rust package registry.</a>
            </div>
            </body></html>"#;
        let results = parse_html_search_results(html);
        assert_eq!(results.len(), 2, "{results:?}");
        assert_eq!(results[0].url, "https://doc.rust-lang.org/book/");
        assert_eq!(results[0].title, "The Rust **Book**");
        assert!(results[0].snippet.contains("official book"), "{results:?}");
        assert_eq!(results[1].url, "https://crates.io/");
        let rendered = render_search_results("rust book", &results);
        assert!(rendered.contains("2 results"), "{rendered}");
        assert!(
            rendered.contains("1. The Rust **Book** — https://doc.rust-lang.org/book/"),
            "{rendered}"
        );
    }

    #[test]
    fn a_page_with_no_recognizable_results_yields_none() {
        assert!(parse_html_search_results("<html><body>nothing</body></html>").is_empty());
    }
}