characterminator 0.1.2

find and eliminate characters outside an allowed set, per file type and per file, so text costs fewer tokens
Documentation
//! The `Read` of a file that is not plain text: bytes that are not
//! UTF-8 (V66, B17) and a file past the 16 MiB cap (V102). A child of
//! `adapter_test.rs`, reaching its fixtures as `super`.

use super::{answer_in, read_payload, tree};

/// The answer to a `Read` of one file holding raw `bytes`.
fn read_of_bytes(name: &str, ctrm: &str, bytes: &[u8]) -> String {
    let Some(root) = tree(name, &[(".ctrm", ctrm)]) else {
        return String::from("(disk refused)");
    };
    let full = root.join("f.rs");
    if std::fs::write(&full, bytes).is_err() {
        return String::from("(disk refused)");
    }
    answer_in(&read_payload(&root, &full.to_string_lossy()), &root)
}

/// B17: a file that is not text is still shown to the model, lossily
/// decoded, so a hazard in it denies the read (V66) -- after a stray
/// `\xff`, after a NUL, and under a `.ctrm` that cannot be applied.
#[test]
fn a_hazard_in_a_file_that_is_not_text_is_denied() {
    let utf8 = b"ok\nlet x = \"\xe2\x80\xae\";\n\xff\n";
    let got = read_of_bytes("ctrm-guard-notutf8", "", utf8);
    assert!(got.contains("f.rs:2:10 U+202E bidi-control"), "{got}");
    let nul = b"a\0\xe2\x80\xae";
    let broken = "*.rs nosuchset";
    for (name, ctrm) in [("ctrm-guard-nul", ""), ("ctrm-guard-bad", broken)] {
        let got = read_of_bytes(name, ctrm, nul);
        assert!(got.contains("f.rs:1:3 U+202E bidi-control"), "{got}");
    }
}

/// V111: `fix` skips a file that is not text, so its denial points at
/// looking (escaped) or asking instead.
#[test]
fn a_denied_binary_read_points_at_cat_v_not_fix() {
    let got = read_of_bytes("ctrm-guard-bin-step", "", b"a\0\xe2\x80\xae");
    assert!(got.contains("view it escaped (`cat -v "), "{got}");
    assert!(got.contains("or ask the user."), "{got}");
    assert!(!got.contains("run `ctrm fix"), "{got}");
}

/// V112: no blocking hazard in an image -- only the NUL, the bytes a
/// binary is made of, and a soft hyphen its lossy decode turned up by
/// chance: the read goes ahead in SILENCE. An agent reads images and
/// PDFs all session, and a note on each is noise nobody reads.
#[test]
fn a_binary_file_without_a_smuggling_hazard_passes_in_silence() {
    let png = b"\x89PNG\r\n\x1a\n\0\0\xff\xc2\xad";
    assert_eq!(read_of_bytes("ctrm-guard-binary", "", png), "");
}

/// A SPARSE file: `head` at byte 0, `tail` at `at`, nothing (NUL, no
/// disk) between. Its read answer, under a `.ctrm` of `* ascii`.
fn read_of_sparse(name: &str, head: &str, at: u64, tail: &str) -> String {
    use std::io::{Seek, SeekFrom, Write};
    let Some(root) = tree(name, &[(".ctrm", "* ascii\n")]) else {
        return String::from("(disk refused)");
    };
    let full = root.join("big.log");
    let written = std::fs::File::create(&full).and_then(|mut f| {
        f.write_all(head.as_bytes())?;
        f.seek(SeekFrom::Start(at))?;
        f.write_all(tail.as_bytes())
    });
    if written.is_err() {
        return String::from("(disk refused)");
    }
    answer_in(&read_payload(&root, &full.to_string_lossy()), &root)
}

/// A TEXT file: `head`, then `a` up to byte `to`, then `tail`, judged
/// under `* any` -- so the only findings are hazards.
fn read_of_long(name: &str, head: &str, to: u64, tail: &str) -> String {
    use std::io::Write;
    let Some(root) = tree(name, &[(".ctrm", "* any\n")]) else {
        return String::from("(disk refused)");
    };
    let full = root.join("long.txt");
    let fill = usize::try_from(to).unwrap_or(0).saturating_sub(head.len());
    let written = std::fs::File::create(&full).and_then(|mut f| {
        f.write_all(head.as_bytes())?;
        f.write_all(&vec![b'a'; fill])?;
        f.write_all(tail.as_bytes())
    });
    if written.is_err() {
        return String::from("(disk refused)");
    }
    answer_in(&read_payload(&root, &full.to_string_lossy()), &root)
}

/// V102: past the 16 MiB cap only a prefix is judged. A hazard OUTSIDE
/// it is not found, so the read passes, never in silence: the note says
/// the rest was not judged.
#[test]
fn a_file_over_the_cap_is_judged_by_its_prefix_and_says_so() {
    let past = super::super::CAP.saturating_add(10);
    let got = read_of_long("ctrm-guard-cap-tail", "plain\n", past, "\u{202E}");
    assert!(!got.contains("permissionDecision"), "{got}");
    assert!(got.contains("16 MiB cap"), "{got}");
    assert!(got.contains("the rest was NOT"), "{got}");
}

/// B64: an early newline, then one line running past the cap. The cut
/// used to land after that newline and judge `x\n` alone; the whole
/// prefix is judged now, so the override on line 2 denies.
#[test]
fn an_override_after_the_last_newline_in_the_prefix_still_denies() {
    let past = super::super::CAP.saturating_add(10);
    let got = read_of_long("ctrm-guard-cap-b64", "x\n\u{202E}", past, "");
    assert!(got.contains(r#""permissionDecision":"deny""#), "{got}");
    assert!(got.contains("long.txt:2:1 U+202E"), "{got}");
}

/// B64: no newline at all, and the cap lands inside an emoji sequence --
/// after a family's joiner, or inside a flag's tags. The cut backs off
/// the unfinished sequence, so it makes no hazard: no denial for size,
/// no joiner noted, only the "rest was not judged" note.
#[test]
fn a_cap_inside_an_emoji_sequence_makes_no_hazard() {
    let cap = super::super::CAP;
    let family = "\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}";
    let flag =
        "\u{1F3F4}\u{E0067}\u{E0062}\u{E0065}\u{E006E}\u{E0067}\u{E007F}";
    // 14 bytes: the man, a joiner, the woman, a joiner. 12: flag + 2 tags.
    for (name, tail, inside) in [("zwj", family, 14), ("tag", flag, 12)] {
        let to = cap.saturating_sub(inside);
        let got = read_of_long(&format!("ctrm-guard-cap-{name}"), "", to, tail);
        assert!(!got.contains("permissionDecision"), "{got}");
        assert!(!got.contains("invisible"), "{got}");
        assert!(got.contains("the rest was NOT"), "{got}");
    }
}

/// V112: a binary over the cap passes in silence too; the "rest was not
/// judged" note is for text.
#[test]
fn a_binary_over_the_cap_passes_in_silence() {
    let past = super::super::CAP.saturating_add(10);
    let got = read_of_sparse("ctrm-guard-cap-binary", "\u{0}PNG", past, "x");
    assert_eq!(got, "");
}

/// A hazard INSIDE the prefix still denies: the cap bounds memory, it
/// opens no door.
#[test]
fn a_hazard_in_the_prefix_of_a_capped_file_still_denies() {
    let past = super::super::CAP.saturating_add(10);
    let got =
        read_of_sparse("ctrm-guard-cap-head", "a\u{202E}b\n", past, "x\n");
    assert!(got.contains(r#""permissionDecision":"deny""#), "{got}");
    assert!(got.contains("big.log:1:2 U+202E"), "{got}");
}

/// The cut lands after the last whole UTF-8 character inside the cap,
/// never at a newline before it (B64), so it never splits one; a file
/// within the cap is read whole.
#[test]
fn the_prefix_is_cut_at_a_whole_character() {
    use super::super::cut;
    assert_eq!(cut(b"ab\ncd\nef", 7), 7);
    assert_eq!(cut("x\n\u{202E}aaaa".as_bytes(), 8), 8);
    // U+2014 is 3 bytes at 1..4; a cap of 3 would split it.
    assert_eq!(cut("a\u{2014}b".as_bytes(), 3), 1);
    assert_eq!(cut(b"abcdef", 4), 4);
    let root = tree("ctrm-guard-cap-small", &[("s.md", "ab\ncd\n")]);
    let file = root.unwrap_or_default().join("s.md");
    let whole = super::super::loaded(&file, 6).map(|(b, w)| (b.len(), w));
    assert_eq!(whole, Some((6, true)));
    let capped = super::super::loaded(&file, 5).map(|(b, w)| (b.len(), w));
    assert_eq!(capped, Some((5, false)));
}