pdf-redact 1.2.0

//! Text that quotes the searched term outside the page content.
//!
//! The area filter removes what is drawn under a box. A comment, a form value,
//! a bookmark title, a named destination, an attachment name, `/Alt` or
//! `/ActualText` that merely *quotes* the term is not under any box, so a
//! search redaction used to leave it in the file. This sweep runs after the
//! page work of [`crate::search_and_redact`] and masks the match in every text
//! string of the document, drops the name-tree entries and legacy destinations
//! whose name holds it, and replaces the XMP packets and rich-text values that
//! hold it.

// Copyright (c) 2026 Innovation Trigger B.V.
//
// PDFluent is available under two licences, at your option: the GNU AGPLv3, or
// the PDFluent Commercial Licence. See the LICENSE file in this repository --
// that file travels with the copy you received, which a URL does not.

use crate::error::RedactError;
use lopdf::{Dictionary, Document, Object, ObjectId, StringFormat};
use std::collections::HashSet;
use std::ops::Range;

/// Deepest name tree walked; a real tree is a few levels.
const MAX_TREE_DEPTH: usize = 64;

/// Finds the byte ranges of the searched term in a text.
pub(crate) type Finder<'a> = &'a dyn Fn(&str) -> Vec<Range<usize>>;

/// Smallest and largest key under a name-tree node.
type KeyRange = (Vec<u8>, Vec<u8>);

/// Keys that hold user-visible or descriptive text. Only strings under these
/// keys are masked, so a broad pattern such as `\d+` cannot reach dates (`/M`,
/// `/CreationDate`), `/ID`, signature data or other structural strings.
const TEXT_KEYS: &[&[u8]] = &[
    b"Contents",
    b"RC",
    b"Subj",
    b"T",
    b"TU",
    b"TM",
    b"DS",
    b"V",
    b"DV",
    b"Opt",
    b"Title",
    b"Alt",
    b"ActualText",
    b"E",
    b"F",
    b"UF",
    b"Desc",
    b"P",
    b"Name",
    b"Author",
    b"Subject",
    b"Keywords",
    b"Creator",
    b"Producer",
    // Actions, links and captions.
    b"JS",
    b"URI",
    b"CA",
    b"AC",
    // Signature dictionaries.
    b"Reason",
    b"Location",
    b"ContactInfo",
];

/// Replacement for a masked character.
const MASK: char = '*';

enum Enc {
    Utf16,
    /// UTF-8, with or without the byte order mark the source carried.
    Utf8 {
        bom: bool,
    },
    Latin1,
}

/// Decode a PDF text string. `None` for binary data (control bytes), which a
/// search term is not looked for in.
fn decode(bytes: &[u8]) -> Option<(String, Enc)> {
    if bytes.len() >= 2 && bytes[..2] == [0xFE, 0xFF] {
        let units: Vec<u16> = bytes[2..]
            .chunks_exact(2)
            .map(|c| u16::from_be_bytes([c[0], c[1]]))
            .collect();
        return String::from_utf16(&units).ok().map(|s| (s, Enc::Utf16));
    }
    if bytes
        .iter()
        .any(|&b| b < 0x20 && !matches!(b, b'\t' | b'\n' | b'\r'))
    {
        return None;
    }
    if let Ok(s) = std::str::from_utf8(bytes) {
        let bom = s.starts_with('\u{FEFF}');
        let s = s.strip_prefix('\u{FEFF}').unwrap_or(s);
        return Some((s.to_string(), Enc::Utf8 { bom }));
    }
    Some((bytes.iter().map(|&b| b as char).collect(), Enc::Latin1))
}

fn encode(text: &str, enc: Enc) -> Vec<u8> {
    match enc {
        Enc::Utf16 => {
            let mut out = vec![0xFE, 0xFF];
            out.extend(text.encode_utf16().flat_map(|u| u.to_be_bytes()));
            out
        }
        Enc::Utf8 { bom } => {
            let mut out = if bom {
                vec![0xEF, 0xBB, 0xBF]
            } else {
                Vec::new()
            };
            out.extend_from_slice(text.as_bytes());
            out
        }
        Enc::Latin1 => text.chars().map(|c| c as u32 as u8).collect(),
    }
}

/// `text` with every match replaced by [`MASK`], one per character; `None`
/// when nothing matched.
fn mask(text: &str, find: Finder) -> Option<String> {
    let ranges: Vec<Range<usize>> = find(text).into_iter().filter(|r| r.start < r.end).collect();
    if ranges.is_empty() {
        return None;
    }
    let mut out = String::with_capacity(text.len());
    for (i, c) in text.char_indices() {
        if ranges.iter().any(|r| r.contains(&i)) {
            out.push(MASK);
        } else {
            out.push(c);
        }
    }
    Some(out)
}

fn matches_bytes(bytes: &[u8], find: Finder) -> bool {
    decode(bytes).is_some_and(|(s, _)| find(&s).iter().any(|r| r.start < r.end))
}

/// Strip markup from a rich-text value so a term split by tags still matches.
fn strip_tags(s: &str) -> String {
    let mut out = String::with_capacity(s.len());
    let mut in_tag = false;
    for c in s.chars() {
        match c {
            '<' => in_tag = true,
            '>' if in_tag => in_tag = false,
            _ if !in_tag => out.push(c),
            _ => {}
        }
    }
    out
}

/// Mask a name (`/Dest /Jane#20Doe`) the same way.
fn mask_name(name: &mut Vec<u8>, find: Finder) -> bool {
    let text: String = name.iter().map(|&b| b as char).collect();
    match mask(&text, find) {
        Some(m) => {
            *name = m.chars().map(|c| c as u32 as u8).collect();
            true
        }
        None => false,
    }
}

/// Walk `obj`; strings are masked only while `text` holds, i.e. under a key of
/// [`TEXT_KEYS`].
fn scrub_object(obj: &mut Object, find: Finder, text: bool) {
    match obj {
        Object::Dictionary(d) => scrub_dict(d, find),
        Object::Stream(s) => scrub_dict(&mut s.dict, find),
        Object::Array(a) => a.iter_mut().for_each(|o| scrub_object(o, find, text)),
        Object::String(bytes, fmt) if text => {
            if let Some((text, enc)) = decode(bytes) {
                if let Some(m) = mask(&text, find) {
                    *bytes = encode(&m, enc);
                    *fmt = StringFormat::Literal;
                }
            }
        }
        _ => {}
    }
}

fn scrub_dict(d: &mut Dictionary, find: Finder) {
    let signature = d.has(b"ByteRange");
    for (key, value) in d.iter_mut() {
        if signature && key == b"Contents" {
            continue;
        }
        // A destination named by a /Name value.
        if matches!(key.as_slice(), b"Dest" | b"D") {
            match value {
                Object::Name(n) => {
                    mask_name(n, find);
                    continue;
                }
                // A named destination written as a string.
                Object::String(..) => {
                    scrub_object(value, find, true);
                    continue;
                }
                _ => {}
            }
        }
        scrub_object(value, find, TEXT_KEYS.contains(&key.as_slice()));
    }
}

/// Drop name-tree entries whose key holds the term, then rewrite `/Limits`.
/// Returns the smallest and largest key left under this node.
fn scrub_name_tree_node(
    doc: &mut Document,
    node: Object,
    find: Finder,
    is_root: bool,
    seen: &mut HashSet<ObjectId>,
    depth: usize,
    too_deep: &mut bool,
) -> (Object, Option<KeyRange>) {
    let Object::Dictionary(mut d) = node else {
        return (node, None);
    };
    let mut range: Option<KeyRange> = None;
    let widen = |k: &[u8], r: &mut Option<KeyRange>| match r {
        None => *r = Some((k.to_vec(), k.to_vec())),
        Some((lo, hi)) => {
            if k < lo.as_slice() {
                *lo = k.to_vec();
            }
            if k > hi.as_slice() {
                *hi = k.to_vec();
            }
        }
    };
    if let Ok(Object::Array(pairs)) = d.get(b"Names").cloned() {
        let mut kept = Vec::new();
        for pair in pairs.chunks(2) {
            if let [Object::String(key, _), value] = pair {
                if matches_bytes(key, find) {
                    continue;
                }
                widen(key, &mut range);
                kept.push(Object::String(key.clone(), StringFormat::Literal));
                kept.push(value.clone());
            } else {
                kept.extend(pair.iter().cloned());
            }
        }
        d.set("Names", Object::Array(kept));
    }
    if let Ok(Object::Array(kids)) = d.get(b"Kids").cloned() {
        // Past the limit the subtree is not inspected, and it stays linked
        // into the document: the caller must refuse, not report success.
        if depth >= MAX_TREE_DEPTH {
            if !kids.is_empty() {
                *too_deep = true;
            }
            return (Object::Dictionary(d), range);
        }
        let mut new_kids = Vec::with_capacity(kids.len());
        for kid in kids {
            match kid {
                // A node reachable twice, or from itself, is walked once.
                Object::Reference(id) => {
                    new_kids.push(Object::Reference(id));
                    if !seen.insert(id) {
                        continue;
                    }
                    let Ok(child) = doc.get_object(id).cloned() else {
                        continue;
                    };
                    let (child, child_range) =
                        scrub_name_tree_node(doc, child, find, false, seen, depth + 1, too_deep);
                    doc.objects.insert(id, child);
                    if let Some((lo, hi)) = child_range {
                        widen(&lo, &mut range);
                        widen(&hi, &mut range);
                    }
                }
                direct @ Object::Dictionary(_) => {
                    let (child, child_range) =
                        scrub_name_tree_node(doc, direct, find, false, seen, depth + 1, too_deep);
                    new_kids.push(child);
                    if let Some((lo, hi)) = child_range {
                        widen(&lo, &mut range);
                        widen(&hi, &mut range);
                    }
                }
                other => new_kids.push(other),
            }
        }
        d.set("Kids", Object::Array(new_kids));
    }
    if !is_root {
        match &range {
            Some((lo, hi)) => d.set(
                "Limits",
                Object::Array(vec![
                    Object::String(lo.clone(), StringFormat::Literal),
                    Object::String(hi.clone(), StringFormat::Literal),
                ]),
            ),
            None => {
                d.remove(b"Limits");
            }
        }
    }
    (Object::Dictionary(d), range)
}

fn resolve_dict_slot(doc: &Document, obj: &Object) -> Option<Dictionary> {
    match obj {
        Object::Dictionary(d) => Some(d.clone()),
        Object::Reference(id) => doc.get_dictionary(*id).ok().cloned(),
        _ => None,
    }
}

/// Name trees under the catalog `/Names` and the legacy catalog `/Dests`.
fn scrub_destinations_and_name_trees(doc: &mut Document, find: Finder) -> Result<(), RedactError> {
    let Ok(root_id) = doc.trailer.get(b"Root").and_then(Object::as_reference) else {
        return Ok(());
    };
    let Ok(catalog) = doc.get_dictionary(root_id).cloned() else {
        return Ok(());
    };

    // PDF 1.1 style: a dictionary keyed by destination name.
    if let Ok(slot) = catalog.get(b"Dests") {
        let target = match slot {
            Object::Reference(id) => Some(*id),
            _ => None,
        };
        if let Some(mut dests) = resolve_dict_slot(doc, slot) {
            let doomed: Vec<Vec<u8>> = dests
                .iter()
                .filter(|(k, _)| {
                    let text: String = k.iter().map(|&b| b as char).collect();
                    find(&text).iter().any(|r| r.start < r.end)
                })
                .map(|(k, _)| k.clone())
                .collect();
            if !doomed.is_empty() {
                for k in doomed {
                    dests.remove(&k);
                }
                match target {
                    Some(id) => {
                        doc.objects.insert(id, Object::Dictionary(dests));
                    }
                    None => {
                        if let Ok(Object::Dictionary(c)) = doc.get_object_mut(root_id) {
                            c.set("Dests", Object::Dictionary(dests));
                        }
                    }
                }
            }
        }
    }

    // Name trees: /Names << /Dests tree /EmbeddedFiles tree ... >>.
    let Some(names_slot) = catalog.get(b"Names").ok().cloned() else {
        return Ok(());
    };
    let Some(names) = resolve_dict_slot(doc, &names_slot) else {
        return Ok(());
    };
    let mut rewritten = names.clone();
    let mut seen: HashSet<ObjectId> = HashSet::new();
    let mut too_deep = false;
    for (key, tree) in names.iter() {
        match tree {
            Object::Reference(id) => {
                if !seen.insert(*id) {
                    continue;
                }
                if let Ok(node) = doc.get_object(*id).cloned() {
                    let (node, _) =
                        scrub_name_tree_node(doc, node, find, true, &mut seen, 0, &mut too_deep);
                    doc.objects.insert(*id, node);
                }
            }
            Object::Dictionary(_) => {
                let (node, _) = scrub_name_tree_node(
                    doc,
                    tree.clone(),
                    find,
                    true,
                    &mut seen,
                    0,
                    &mut too_deep,
                );
                rewritten.set(key.clone(), node);
            }
            _ => {}
        }
    }
    if too_deep {
        return Err(RedactError::Other(format!(
            "a name tree is nested deeper than {MAX_TREE_DEPTH} levels and cannot be \
             inspected for the searched text; redaction refused instead of leaving the \
             uninspected names in the file"
        )));
    }
    match names_slot {
        Object::Reference(id) => {
            doc.objects.insert(id, Object::Dictionary(rewritten));
        }
        _ => {
            if let Ok(Object::Dictionary(c)) = doc.get_object_mut(root_id) {
                c.set("Names", Object::Dictionary(rewritten));
            }
        }
    }
    Ok(())
}

/// Remove `/RC` (rich text) that holds the term, as a string or a stream.
/// Markup can split a term (`Jane</b> <b>Doe`), so the tags are stripped before
/// matching; a match drops the value rather than masking half of a document.
fn drop_rich_text(doc: &mut Document, find: Finder) {
    let mut doomed: Vec<ObjectId> = Vec::new();
    for (id, obj) in doc.objects.iter() {
        let dict = match obj {
            Object::Dictionary(d) => d,
            Object::Stream(s) => &s.dict,
            _ => continue,
        };
        let Ok(rc) = dict.get(b"RC") else { continue };
        let text = match rc {
            Object::String(b, _) => decode(b).map(|(s, _)| s),
            Object::Reference(r) => match doc.get_object(*r) {
                Ok(Object::Stream(s)) => Some(
                    String::from_utf8_lossy(&s.get_plain_content().unwrap_or(s.content.clone()))
                        .into_owned(),
                ),
                Ok(Object::String(b, _)) => decode(b).map(|(s, _)| s),
                _ => None,
            },
            _ => None,
        };
        if let Some(t) = text {
            let hit = |s: &str| find(s).iter().any(|r| r.start < r.end);
            if hit(&t) || hit(&strip_tags(&t)) {
                doomed.push(*id);
            }
        }
    }
    for id in doomed {
        match doc.get_object_mut(id) {
            Ok(Object::Dictionary(d)) => {
                d.remove(b"RC");
            }
            Ok(Object::Stream(s)) => {
                s.dict.remove(b"RC");
            }
            _ => {}
        }
    }
}

/// Mask the term in script streams (`/JS` pointing at a stream). The action
/// stays, only the matched text goes.
fn scrub_script_streams(doc: &mut Document, find: Finder) {
    fn collect(obj: &Object, out: &mut Vec<ObjectId>) {
        match obj {
            Object::Dictionary(d) => collect_dict(d, out),
            Object::Stream(s) => collect_dict(&s.dict, out),
            Object::Array(a) => a.iter().for_each(|o| collect(o, out)),
            _ => {}
        }
    }
    fn collect_dict(d: &Dictionary, out: &mut Vec<ObjectId>) {
        for (k, v) in d.iter() {
            if k == b"JS" {
                if let Object::Reference(id) = v {
                    out.push(*id);
                }
            }
            collect(v, out);
        }
    }
    let mut ids = Vec::new();
    for obj in doc.objects.values() {
        collect(obj, &mut ids);
    }
    ids.sort_unstable();
    ids.dedup();
    for id in ids {
        let Ok(Object::Stream(s)) = doc.get_object(id) else {
            continue;
        };
        let plain = s.get_plain_content().unwrap_or_else(|_| s.content.clone());
        let Some((text, enc)) = decode(&plain) else {
            continue;
        };
        let Some(masked) = mask(&text, find) else {
            continue;
        };
        let bytes = encode(&masked, enc);
        if let Ok(Object::Stream(s)) = doc.get_object_mut(id) {
            s.dict.remove(b"Filter");
            s.dict.remove(b"DecodeParms");
            s.dict.set("Length", Object::Integer(bytes.len() as i64));
            s.content = bytes;
        }
    }
}

/// Replace every `/Type /Metadata` packet that holds the term with the stub.
fn replace_metadata_packets(doc: &mut Document, find: Finder, stub: &[u8]) {
    for obj in doc.objects.values_mut() {
        let Object::Stream(s) = obj else { continue };
        let is_meta = matches!(s.dict.get(b"Type"), Ok(Object::Name(n)) if n == b"Metadata");
        if !is_meta {
            continue;
        }
        let plain = s.get_plain_content().unwrap_or_else(|_| s.content.clone());
        let hit = |b: &[u8]| {
            let lossy = String::from_utf8_lossy(b);
            find(&lossy).iter().any(|r| r.start < r.end) || matches_bytes(b, find) || {
                // UTF-16 packets.
                let units: Vec<u16> = b
                    .chunks_exact(2)
                    .map(|c| u16::from_be_bytes([c[0], c[1]]))
                    .collect();
                String::from_utf16(&units).is_ok_and(|t| find(&t).iter().any(|r| r.start < r.end))
            }
        };
        if hit(&plain) || hit(&s.content) {
            s.content = stub.to_vec();
            s.dict.remove(b"Filter");
            s.dict.remove(b"DecodeParms");
            s.dict.set("Length", Object::Integer(stub.len() as i64));
        }
    }
}

/// Mask the term in the inline property lists of marked content
/// (`/Span <</ActualText (..)>> BDC`), which sit inside content streams. Only a
/// stream that holds a match is decoded, edited and written back.
fn scrub_marked_content(doc: &mut Document, find: Finder) {
    let mut ids: Vec<ObjectId> = Vec::new();
    for (id, obj) in doc.objects.iter() {
        let Object::Stream(s) = obj else { continue };
        let form = matches!(s.dict.get(b"Subtype"), Ok(Object::Name(n)) if n == b"Form");
        if form || s.dict.has(b"PatternType") {
            ids.push(*id);
        }
    }
    for page_id in doc.get_pages().values() {
        ids.extend(doc.get_page_contents(*page_id));
    }
    ids.sort_unstable();
    ids.dedup();
    for id in ids {
        let Ok(Object::Stream(s)) = doc.get_object(id) else {
            continue;
        };
        let Ok(plain) = s.get_plain_content() else {
            continue;
        };
        if !plain.windows(2).any(|w| w == b"DC" || w == b"DP") {
            continue;
        }
        let Ok(mut content) = lopdf::content::Content::decode(&plain) else {
            continue;
        };
        let mut changed = false;
        for op in content.operations.iter_mut() {
            if !matches!(op.operator.as_str(), "BDC" | "DP") {
                continue;
            }
            for operand in op.operands.iter_mut() {
                if matches!(operand, Object::Dictionary(_)) {
                    let before = operand.clone();
                    scrub_object(operand, find, false);
                    changed |= *operand != before;
                }
            }
        }
        if !changed {
            continue;
        }
        let Ok(encoded) = content.encode() else {
            continue;
        };
        if let Ok(Object::Stream(s)) = doc.get_object_mut(id) {
            s.dict.remove(b"Filter");
            s.dict.remove(b"DecodeParms");
            s.dict.set("Length", Object::Integer(encoded.len() as i64));
            s.content = encoded;
        }
    }
}

/// Indirect objects that a [`TEXT_KEYS`] entry (or a `/Dest`, `/D` entry)
/// points to, directly or through arrays of references. `scrub_object` sees a
/// bare `Object::Reference` there and cannot follow it, so the final sweep
/// masks these objects as text instead. Returns `(text, destination names)`.
fn indirect_text_targets(doc: &Document) -> (HashSet<ObjectId>, HashSet<ObjectId>) {
    fn follow(doc: &Document, id: ObjectId, out: &mut HashSet<ObjectId>) {
        // Worklist with a visited set: an array that contains itself ends.
        let mut stack = vec![id];
        while let Some(id) = stack.pop() {
            if !out.insert(id) {
                continue;
            }
            if let Ok(Object::Array(items)) = doc.get_object(id) {
                collect_refs(items, &mut stack);
            }
        }
    }
    fn collect_refs(items: &[Object], stack: &mut Vec<ObjectId>) {
        for item in items {
            match item {
                Object::Reference(r) => stack.push(*r),
                Object::Array(inner) => collect_refs(inner, stack),
                _ => {}
            }
        }
    }
    fn value_refs(value: &Object, stack: &mut Vec<ObjectId>) {
        match value {
            Object::Reference(r) => stack.push(*r),
            Object::Array(items) => collect_refs(items, stack),
            _ => {}
        }
    }
    fn visit(obj: &Object, text: &mut Vec<ObjectId>, dests: &mut Vec<ObjectId>) {
        match obj {
            Object::Dictionary(d) => visit_dict(d, text, dests),
            Object::Stream(s) => visit_dict(&s.dict, text, dests),
            Object::Array(items) => items.iter().for_each(|o| visit(o, text, dests)),
            _ => {}
        }
    }
    fn visit_dict(d: &Dictionary, text: &mut Vec<ObjectId>, dests: &mut Vec<ObjectId>) {
        let signature = d.has(b"ByteRange");
        for (key, value) in d.iter() {
            if signature && key == b"Contents" {
                continue;
            }
            if matches!(key.as_slice(), b"Dest" | b"D") {
                value_refs(value, dests);
                value_refs(value, text);
            } else if TEXT_KEYS.contains(&key.as_slice()) {
                value_refs(value, text);
            }
            visit(value, text, dests);
        }
    }
    let (mut text_roots, mut dest_roots) = (Vec::new(), Vec::new());
    for obj in doc.objects.values() {
        visit(obj, &mut text_roots, &mut dest_roots);
    }
    let mut text = HashSet::new();
    for id in text_roots {
        follow(doc, id, &mut text);
    }
    let dests: HashSet<ObjectId> = dest_roots.into_iter().collect();
    (text, dests)
}

/// Scrub the term out of everything that quotes it besides page content.
///
/// Fails, leaving `doc` in an unspecified state, when part of the document
/// cannot be inspected; callers work on a copy and discard it on error.
pub(crate) fn scrub_sidecar_text(
    doc: &mut Document,
    find: Finder,
    metadata_stub: &[u8],
) -> Result<(), RedactError> {
    scrub_destinations_and_name_trees(doc, find)?;
    drop_rich_text(doc, find);
    scrub_marked_content(doc, find);
    scrub_script_streams(doc, find);
    // Taken before the sweep: it follows the references the sweep rewrites.
    let (text_ids, dest_ids) = indirect_text_targets(doc);
    for (id, obj) in doc.objects.iter_mut() {
        if dest_ids.contains(id) {
            if let Object::Name(n) = obj {
                mask_name(n, find);
                continue;
            }
        }
        scrub_object(obj, find, text_ids.contains(id));
    }
    replace_metadata_packets(doc, find, metadata_stub);
    // Entries and streams the sweep detached still hold the term.
    crate::area_filter::drop_unreachable(doc);
    Ok(())
}