semlith 0.34.0

//! The credential content scan: the table, what it refuses at index time, and
//! `semlith scan` over a store built before the rule existed.
//!
//! A refused file is one whose text never reaches the embedder, so most of
//! this needs no model. The two that assert a file was *indexed* do, and say
//! so.

use semlith::{FileOutcome, Semlith};
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};

fn store_at(dir: &Path) -> Semlith {
    let mut s = Semlith::open(dir, None).unwrap();
    s.quiet = true;
    s
}

fn write(dir: &Path, name: &str, body: &str) -> PathBuf {
    let path = dir.join(name);
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent).unwrap();
    }
    fs::write(&path, body).unwrap();
    path
}

/// Index `corpus` and return the report with every `(path, outcome, why)`.
fn run(
    store: &Path,
    corpus: &Path,
    secrets: bool,
) -> (
    semlith::IndexReport,
    Vec<(PathBuf, FileOutcome, Option<String>)>,
) {
    let mut s = store_at(store);
    s.boundary = semlith::Boundary {
        roots: None,
        allow_secrets: secrets,
    };
    let mut events = Vec::new();
    let report = s
        .index_paths(&[corpus.to_path_buf()], |path, p| {
            events.push((path.to_path_buf(), p.outcome, p.why.clone()))
        })
        .unwrap();
    (report, events)
}

/// The row of `SHAPES` whose kind contains `kind`.
fn row(kind: &str) -> usize {
    semlith::filter::SHAPES
        .iter()
        .position(|s| s.kind.contains(kind))
        .unwrap()
}

/// A live-looking value for the row, built now. The source never holds one:
/// from 0.30.0 a live-looking literal here would refuse this file itself.
fn live(kind: &str) -> String {
    semlith::keyscan::forge(row(kind))
}

/// Twenty random characters of a generated key, for the generic rule.
fn random20() -> String {
    live("GitHub token")[4..24].to_string()
}

/// A `.txt` per shape, named after nothing in the deny-list so the only rule
/// that can refuse it is the content scan.
fn credential_files(corpus: &Path) {
    for i in 0..semlith::filter::SHAPES.len() {
        write(
            corpus,
            &format!("config{i}.txt"),
            &format!(
                "# configuration\nvalue = \"{}\"\nother = 1\n",
                semlith::keyscan::forge(i)
            ),
        );
    }
    write(
        corpus,
        "generic.txt",
        &format!("# configuration\napi_key = \"{}\"\n", random20()),
    );
}

/// The table is only as good as the fact that every row means what it says.
#[test]
fn every_shape_matches_its_example_and_not_its_near_miss() {
    for (i, shape) in semlith::filter::SHAPES.iter().enumerate() {
        // The example matches the pattern and is a declared test dummy.
        let example = semlith::keyscan::scan("", shape.example);
        assert!(
            !example.is_empty() && !semlith::keyscan::refuses(&example),
            "{}: the pattern {:?} should match its example {:?} as a dummy: {example:?}",
            shape.kind,
            shape.pattern,
            shape.example
        );
        // A value built now in the row's shape is refused.
        assert!(
            semlith::filter::scan_text(&semlith::keyscan::forge(i)).is_some(),
            "{}: a live-shaped value was let through",
            shape.kind
        );
        assert!(
            !shape.example.is_empty() && !shape.near_miss.is_empty(),
            "{}: a row without both strings is a row nothing checks",
            shape.kind
        );
        assert!(
            semlith::filter::scan_text(shape.near_miss).is_none(),
            "{}: the near miss {:?} matched, so the pattern is too wide",
            shape.kind,
            shape.near_miss
        );
    }
}

/// The generic rule needs both halves. A key-like name with a placeholder is
/// not a credential, and a high-entropy blob with no name beside it is a test
/// fixture or a hash.
#[test]
fn the_generic_rule_needs_the_name_and_the_entropy_together() {
    let value = random20();
    assert!(semlith::filter::scan_text(&format!("api_key = \"{value}\"")).is_some());
    // Entropy without a key-like name.
    assert!(semlith::filter::scan_text(&format!("digest = \"{value}\"")).is_none());
    // A key-like name with no entropy behind it.
    assert!(semlith::filter::scan_text("password = \"aaaaaaaaaaaaaaaaaaaaaa\"").is_none());
    // Short enough to be a real password and not worth refusing a file for.
    assert!(semlith::filter::scan_text("password = \"hunter2\"").is_none());
}

/// The line number is the useful half of the finding.
#[test]
fn the_finding_names_the_line() {
    let text = format!("one\ntwo\nthree\n{}\n", live("GitHub token"));
    let found = semlith::filter::scan_text(&text).expect("a GitHub token");
    assert_eq!(found.line, 4);
    assert!(found.kind.contains("GitHub"));
}

/// The rule the whole feature stands on: the scan's own output must never
/// contain the thing it found.
#[test]
fn no_reason_ever_quotes_the_credential() {
    for (i, shape) in semlith::filter::SHAPES.iter().enumerate() {
        let value = semlith::keyscan::forge(i);
        let found = semlith::filter::scan_text(&value).expect("a live-shaped value");
        let reason = found.reason();
        for window in value
            .as_bytes()
            .windows(8)
            .map(|w| String::from_utf8_lossy(w).into_owned())
        {
            assert!(
                !reason.contains(&window),
                "{}: the reason {reason:?} quotes {window:?} out of the credential",
                shape.kind
            );
        }
    }
}

/// Every shape, as a file, through a real index run.
#[test]
fn a_credential_in_an_ordinary_file_is_refused_at_index_time() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    credential_files(corpus.path());

    let (report, events) = run(store.path(), corpus.path(), false);

    let expected = semlith::filter::SHAPES.len() + 1;
    assert_eq!(
        report.refused.len(),
        expected,
        "every credential fixture should have been refused: {:?}",
        report.refused
    );
    assert_eq!(report.indexed, 0);
    for (path, outcome, why) in &events {
        assert_eq!(
            *outcome,
            FileOutcome::Refused,
            "{} was not refused",
            path.display()
        );
        let why = why.as_deref().unwrap();
        assert!(
            why.contains("line"),
            "the reason should name the line: {why}"
        );
    }
}

/// A cleaned twin of each fixture is ordinary text, and a release that refused
/// it would be one that stopped indexing code.
#[test]
fn the_cleaned_twins_are_not_refused() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    for (i, shape) in semlith::filter::SHAPES.iter().enumerate() {
        write(
            corpus.path(),
            &format!("clean{i}.txt"),
            &format!(
                "# configuration\nvalue = \"{}\"\nother = 1\n",
                shape.near_miss
            ),
        );
    }

    let (report, _) = run(store.path(), corpus.path(), false);
    assert!(
        report.refused.is_empty(),
        "a near miss is not a credential: {:?}",
        report.refused
    );
}

/// The scan reads what a document reader produced, not only what a text file
/// holds. A `.docx` is a zip of XML and the key is in its body.
#[test]
fn a_credential_inside_a_document_is_refused_on_the_readers_text() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    write_docx(
        &corpus.path().join("handover.docx"),
        &format!("Access key {}, please rotate it.", live("AWS")),
    );

    let (report, _) = run(store.path(), corpus.path(), false);

    assert_eq!(
        report.refused.len(),
        1,
        "the reader's text should have been scanned: {report:?}"
    );
    assert!(report.refused[0].1.contains("AWS"));
}

/// An image is not text and is never scanned, even when its bytes happen to
/// spell a credential's prefix.
#[test]
fn an_image_is_not_scanned() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    // Not a decodable image, which is enough: what is asserted is that the
    // content scan never sees it, and a scan that ran would refuse it.
    let mut bytes = b"\x89PNG\r\n\x1a\n".to_vec();
    bytes.extend(live("AWS").as_bytes());
    fs::write(corpus.path().join("shot.png"), bytes).unwrap();

    let (report, _) = run(store.path(), corpus.path(), false);
    assert!(
        report.refused.is_empty(),
        "an image must not be scanned as text: {:?}",
        report.refused
    );
}

/// `--include-secrets` is the user saying they meant it, and the count is
/// semlith saying what that cost.
#[test]
#[ignore = "downloads an embedding model on first run"]
fn include_secrets_indexes_them_and_says_how_many() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    credential_files(corpus.path());

    let (report, _) = run(store.path(), corpus.path(), true);

    assert!(report.refused.is_empty(), "{:?}", report.refused);
    assert_eq!(report.secrets_indexed, semlith::filter::SHAPES.len() + 1);
    assert_eq!(report.indexed, report.secrets_indexed);
}

/// A file that was clean when it was indexed and is not any more leaves the
/// store on the next run, rather than being left behind because the new rule
/// only stops new files.
#[test]
#[ignore = "downloads an embedding model on first run"]
fn a_file_that_gains_a_token_is_evicted_on_the_next_run() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    write(
        corpus.path(),
        "deploy.md",
        "Deployment runs on Tuesdays after the migration has finished.\n",
    );

    let (first, _) = run(store.path(), corpus.path(), false);
    assert_eq!(first.indexed, 1);
    assert!(
        !store_at(store.path())
            .search("deployment on tuesdays", 5)
            .unwrap()
            .is_empty()
    );

    write(
        corpus.path(),
        "deploy.md",
        &format!(
            "Deployment runs on Tuesdays.\ntoken = \"{}\"\n",
            live("GitHub token")
        ),
    );
    let (second, _) = run(store.path(), corpus.path(), false);

    assert_eq!(second.refused.len(), 1);
    assert!(
        second.refused[0].1.contains("removed from this store"),
        "the refusal should say the earlier contents went: {:?}",
        second.refused[0].1
    );
    assert!(
        store_at(store.path())
            .search("deployment on tuesdays", 5)
            .unwrap()
            .is_empty(),
        "the old contents are still searchable after the file was refused"
    );
    // #148: and not merely unsearchable. The evicted text is gone from the
    // database file and its log, not left in freed pages or FTS5 segments.
    for name in ["store.db", "store.db-wal"] {
        let bytes = fs::read(store.path().join(name)).unwrap_or_default();
        for needle in ["after the migration", "Tuesdays after"] {
            assert!(
                !bytes.windows(needle.len()).any(|w| w == needle.as_bytes()),
                "{name} still holds {needle:?} after the eviction"
            );
        }
    }
}

/// The path for a store indexed before any of this existed.
#[test]
#[ignore = "downloads an embedding model on first run"]
fn scan_finds_what_todays_rules_would_refuse_and_forget_evicts_it() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    write(
        corpus.path(),
        "notes.md",
        "Nothing secret here, only the release notes.\n",
    );
    // Indexed with the scan off, which is what a 0.18.0 store is.
    write(
        corpus.path(),
        "handover.md",
        &format!("The key is {} until Friday.\n", live("GitHub token")),
    );
    let (report, _) = run(store.path(), corpus.path(), true);
    assert_eq!(report.indexed, 2);

    let mut s = store_at(store.path());
    let found = s.scan().unwrap();
    assert_eq!(found.len(), 1, "{found:?}");
    assert!(found[0].path.ends_with("handover.md"));
    assert!(found[0].why.contains("GitHub"));

    let gone = s.forget_findings(&found).unwrap();
    assert_eq!(gone, 1);
    assert!(
        s.scan().unwrap().is_empty(),
        "a second scan of a cleaned store finds nothing"
    );
}

/// The smallest `.docx` a reader will take: a zip holding one
/// `word/document.xml`.
fn write_docx(path: &Path, body: &str) {
    let file = fs::File::create(path).unwrap();
    let mut zip = zip::ZipWriter::new(file);
    let options: zip::write::FileOptions<()> =
        zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Deflated);
    zip.start_file("word/document.xml", options).unwrap();
    write!(
        zip,
        "<?xml version=\"1.0\"?><w:document \
         xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\">\
         <w:body><w:p><w:r><w:t>{body}</w:t></w:r></w:p></w:body></w:document>"
    )
    .unwrap();
    zip.finish().unwrap();
}

/// The false-positive audit, as a command rather than a paragraph in a record.
///
/// Point it at a real tree and it prints every file the content scan would
/// refuse, with the kind and the line, reading each file exactly as the index
/// loop does — through `chunk::extract`, so a document reader's text is
/// scanned the way it is in a run. It embeds nothing, so a corpus of any size
/// costs seconds.
///
/// ```sh
/// SEMLITH_SCAN_AUDIT=~/work/some-repo cargo test --test scan audit -- --ignored --nocapture
/// ```
///
/// Ignored because it needs a corpus, not because it is slow. 0.19.0 ran it
/// over two trees before the release proceeded, and the first run is what found
/// the `${SEMLITH_AGENT_KEY}` placeholder rule.
#[test]
#[ignore = "needs SEMLITH_SCAN_AUDIT pointing at a corpus"]
fn audit_a_corpus_for_false_positives() {
    let Ok(root) = std::env::var("SEMLITH_SCAN_AUDIT") else {
        // Nothing to audit is not a failure. This is a tool with a test's
        // shape: it takes a corpus somebody chose, and with none named there
        // is no assertion it could make. Panicking here made the whole
        // ignored suite red for anybody running it as a set, which is the
        // suite a release is verified with.
        eprintln!("skipped: set SEMLITH_SCAN_AUDIT to a tree to audit it for false positives");
        return;
    };
    let root = PathBuf::from(root);
    let mut walked = 0usize;
    let mut refused = 0usize;
    let mut texts: Vec<String> = Vec::new();
    let mut stack = vec![root.clone()];
    while let Some(dir) = stack.pop() {
        let Ok(entries) = fs::read_dir(&dir) else {
            continue;
        };
        for entry in entries.flatten() {
            let path = entry.path();
            let name = entry.file_name().to_string_lossy().into_owned();
            if path.is_dir() {
                // The directories a walk would never yield anyway.
                if !matches!(
                    name.as_str(),
                    ".git" | "target" | "node_modules" | ".semlith" | "dist" | "build"
                ) {
                    stack.push(path);
                }
                continue;
            }
            let Ok(bytes) = fs::read(&path) else { continue };
            if bytes.len() as u64 > 8 * 1024 * 1024 {
                continue;
            }
            let Ok(text) = semlith::chunk::extract(&path, &bytes) else {
                continue;
            };
            walked += 1;
            if let Some(found) = semlith::filter::scan_text(&text) {
                refused += 1;
                println!("{}: {}", path.display(), found.reason());
            }
            texts.push(text);
        }
    }
    println!("audited {walked} readable files under {}", root.display());
    println!("{refused} would be refused");

    // What the scan itself costs, which is the number the throughput claim
    // rests on. An end-to-end index run is almost entirely the embedder, so a
    // wall-clock comparison of two runs measures the model rather than this.
    let mut total = 0usize;
    let start = std::time::Instant::now();
    for text in &texts {
        total += text.len();
        std::hint::black_box(semlith::filter::scan_text(text));
    }
    let spent = start.elapsed();
    println!(
        "the scan itself: {:.1} ms over {:.1} MiB of text ({:.0} MiB/s)",
        spent.as_secs_f64() * 1000.0,
        total as f64 / (1024.0 * 1024.0),
        (total as f64 / (1024.0 * 1024.0)) / spent.as_secs_f64()
    );
}

/// The two shapes the false-positive audit found in real trees, as a gate.
///
/// Both were matches of the one rule with no prefix behind it, and both are
/// things a credential is never written as. Without these the rule refuses a
/// page documenting how not to write a key down, and a `token=` on one line
/// with a quote two lines later.
#[test]
fn a_placeholder_is_not_a_credential() {
    for value in [
        r#"Authorization = "Bearer ${SEMLITH_AGENT_KEY}""#,
        r#"api_key = "{{ openai_api_key_goes_here }}""#,
        r#"password = "<your-database-password-here>""#,
        r#"token: "SEMLITH_AGENT_KEY""#,
    ] {
        assert!(
            semlith::filter::scan_text(value).is_none(),
            "a placeholder was refused as a credential: {value}"
        );
    }
}

/// The value may not cross a line. `split_once("/?token=")` and a quote two
/// lines later are not an assignment; they are code with a colon in it.
#[test]
fn a_generic_match_does_not_span_lines() {
    let code = "let (port, query) = rest.split_once(\"/?token=\").unwrap_or_else(|| {\n\
                    panic!(\"the URL does not carry a token in the documented shape: {url}\")\n\
                });\n";
    assert!(
        semlith::filter::scan_text(code).is_none(),
        "a multi-line match was treated as an assignment"
    );
}

/// A refusal that is about this caller rather than about the file must not
/// evict what another caller already indexed.
///
/// An agent confined to one root that asks to index a path outside it would
/// otherwise be able to delete rows the command line put there — a caller who
/// may not read a file deleting it.
#[test]
fn a_boundary_refusal_evicts_nothing() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    let outside = tempfile::tempdir().unwrap();
    write(corpus.path(), "notes.md", "Release notes for the API.\n");
    let elsewhere = write(outside.path(), "plans.md", "The plans.\n");

    // Already in the store, put there by a caller with no boundary — written
    // rather than embedded, because what is under test is the refusal and not
    // the embedder.
    let s = store_at(store.path());
    let key = elsewhere.to_string_lossy().into_owned();
    semlith::store::read_only(s.db(), false).unwrap();
    let file = semlith::store::insert_file(s.db(), &key, "hash", 11, 0).unwrap();
    semlith::store::insert_chunk(s.db(), file, 0, 1, 1, "The plans.").unwrap();
    semlith::store::read_only(s.db(), true).unwrap();
    let before = semlith::store::all_paths(s.db()).unwrap();
    assert_eq!(before.len(), 1, "{before:?}");
    drop(s);

    // Whether this test has anything to say depends on where the operating
    // system puts temporary directories. `within_boundary` admits anything
    // under the user's home, and on Windows the temp directory is under it —
    // so the path this test calls "outside" is inside the boundary there, is
    // indexed rather than refused, and there is no refusal to assert about.
    // Checked before the run rather than after, so that platform does not pay
    // for an embedding it cannot learn anything from. The invariant itself is
    // pinned by `only_a_refusal_about_the_file_itself_is_marked_for_eviction`,
    // which touches no filesystem and runs everywhere.
    if semlith::filter::within_boundary(&elsewhere, &[corpus.path().to_path_buf()]) {
        return;
    }

    // Now as an agent confined to the corpus, asking for the path outside it.
    let mut s = store_at(store.path());
    s.boundary = semlith::Boundary::within(vec![corpus.path().to_path_buf()]);
    let report = s.index_paths(&[elsewhere], |_, _| {}).unwrap();

    assert_eq!(report.refused.len(), 1, "{:?}", report.refused);
    let why = &report.refused[0].1;
    assert!(why.contains("outside"), "{why:?}");
    assert!(
        !why.contains("removed from this store"),
        "a boundary refusal claimed to have evicted something"
    );
    let after = semlith::store::all_paths(s.db()).unwrap();
    assert_eq!(
        after, before,
        "a refusal about the caller deleted what another caller indexed"
    );
}

/// The flag that decides whether a refusal evicts, pinned without touching a
/// filesystem.
///
/// `Boundary::refuses` resolves nothing on disk — `canonical` falls back to the
/// path it was given — so a synthetic path outside any home exercises the
/// boundary branch identically on all three platforms.
#[test]
fn only_a_refusal_about_the_file_itself_is_marked_for_eviction() {
    let home = PathBuf::from(if cfg!(windows) {
        r"C:\Users\nobody"
    } else {
        "/home/nobody"
    });
    let root = home.join("work").join("api");
    let outside = PathBuf::from(if cfg!(windows) {
        r"C:\elsewhere\somebody-elses\notes.md"
    } else {
        "/elsewhere/somebody-elses/notes.md"
    });

    let agent = semlith::Boundary::within(vec![root.clone()]);
    let refusal = agent
        .refuses(&outside, false, Some(&home))
        .expect("a path outside the roots and outside the home is refused");
    assert!(refusal.why.contains("outside"), "{:?}", refusal.why);
    assert!(
        !refusal.credential,
        "a refusal about the caller must not evict what another caller indexed"
    );

    // The other kind, for contrast: this one is about the file, and does evict.
    let named = semlith::Boundary::default()
        .refuses(&root.join(".env"), true, Some(&home))
        .expect("a credential is refused by name");
    assert!(named.credential, "{:?}", named.why);
}

/// Each declared dummy rule lets its own case through, and one live-looking
/// match beside a dummy still refuses the file (2.1).
#[test]
fn each_dummy_rule_passes_its_case_and_a_live_neighbour_still_refuses() {
    let aws_id = row("AWS");
    let example = semlith::filter::SHAPES[aws_id].example;
    // (a) a published documentation example.
    assert!(semlith::filter::scan_text(&format!("key {example}")).is_none());
    // (b) a marker in the body, a run of X, one character repeated.
    for body in ["FAKEFAKEFAKEFAKE", "XXXXXXXXXXXXXXXX", "AAAAAAAAAAAAAAAA"] {
        assert!(
            semlith::filter::scan_text(&format!("AKIA{body}")).is_none(),
            "AKIA{body}"
        );
    }
    // A marker in the text around a key is not in its body.
    let key = live("AWS");
    assert!(semlith::filter::scan_text(&format!("# FAKE test key\n{key}")).is_some());
    // (c) a private-key header with no body, and one with a real body.
    assert!(
        semlith::filter::scan_text(
            "-----BEGIN RSA PRIVATE KEY-----\n-----END RSA PRIVATE KEY-----"
        )
        .is_none()
    );
    assert!(semlith::filter::scan_text(&live("private key")).is_some());
    // One dummy and one live id: refused.
    assert!(semlith::filter::scan_text(&format!("{example}\n{key}\n")).is_some());
}

/// The no-prefix rule: a secret-sounding name assigned a long random literal
/// refuses the file, quoted or not; a dummy or a low-entropy value does not
/// (2.2).
#[test]
fn a_secret_with_no_prefix_is_refused_by_its_name_and_randomness() {
    let value = format!("{}{}", random20(), random20());
    assert_eq!(value.len(), 40);
    assert!(semlith::filter::scan_text(&format!("AWS_SECRET_ACCESS_KEY={value}\n")).is_some());
    assert!(semlith::filter::scan_text(&format!("export DB_TOKEN={value}\n")).is_some());
    assert!(
        semlith::filter::scan_text(
            "AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\n"
        )
        .is_none()
    );
    assert!(semlith::filter::scan_text("PASSWORD=changeme\n").is_none());
    // Code is not an assignment of a literal.
    assert!(
        semlith::filter::scan_text("let token = compute_the_session_token(input);\n").is_none()
    );
}

/// A fixture tree with one file for each not-indexed class, plus a file whose
/// only match is a test dummy and one ordinary file.
fn every_class(corpus: &Path) {
    write(
        corpus,
        "config.txt",
        &format!("# settings\nkey = \"{}\"\n", live("AWS")),
    );
    write(corpus, ".env", &format!("TOKEN={}\n", live("GitHub token")));
    write(corpus, "big.txt", &"x".repeat(9 * 1024 * 1024));
    fs::write(corpus.join("blob.dat"), [0u8, 1, 2, 0, 255, 0, 7]).unwrap();
    write(corpus, ".gitignore", "skipped.txt\n");
    write(corpus, "skipped.txt", "ignored by the user's own rule\n");
    let example = semlith::filter::SHAPES[row("AWS")].example;
    write(
        corpus,
        "docs.md",
        &format!("# Keys\nUse {example} in examples.\n"),
    );
    write(corpus, "lib.rs", "pub fn plain() -> u32 { 1 }\n");
}

/// 2.3: every class is listed with its rule, the dummy is let through, and
/// the list outlives the process that wrote it.
#[test]
#[ignore = "downloads an embedding model on first run"]
fn the_not_indexed_list_has_every_class_and_survives_reopening() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    every_class(corpus.path());
    let (report, _) = run(store.path(), corpus.path(), false);
    assert_eq!(report.dummies_indexed, 1, "{report:?}");

    let s = store_at(store.path());
    let rows = semlith::store::refusals(s.db()).unwrap();
    let class_of = |name: &str| {
        rows.iter()
            .find(|r| r.path.ends_with(name))
            .map(|r| r.class.clone())
            .unwrap_or_else(|| panic!("{name} is not listed: {rows:#?}"))
    };
    assert_eq!(class_of("config.txt"), "content");
    assert_eq!(class_of(".env"), "credential");
    assert_eq!(class_of("big.txt"), "policy");
    assert_eq!(class_of("blob.dat"), "unindexable");
    assert_eq!(class_of("skipped.txt"), "excluded");
    assert_eq!(class_of("docs.md"), "dummy");
    assert!(rows.iter().all(|r| !r.rule.is_empty()), "{rows:#?}");
    let content = rows.iter().find(|r| r.class == "content").unwrap();
    assert!(content.confidence.is_some_and(|c| c >= 70), "{content:?}");
    drop(s);
    // Survives the process: a fresh open reads the same rows.
    let again = store_at(store.path());
    assert_eq!(
        semlith::store::refusals(again.db()).unwrap().len(),
        rows.len()
    );
}

/// 2.5 and 2.6: accepted with redaction, the file is indexed with its value
/// replaced at the same line; a comment edit keeps it; a new key refuses it
/// again and evicts it; as-is indexes the value; revoke evicts and relists.
#[test]
#[ignore = "downloads an embedding model on first run"]
fn an_accepted_file_is_indexed_redacted_and_a_new_secret_refuses_it_again() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    let key = live("AWS");
    let file = write(
        corpus.path(),
        "deploy.txt",
        &format!("one\nkey = \"{key}\"\nthree\n"),
    );
    run(store.path(), corpus.path(), false);
    let path = semlith::canonical(&file).to_string_lossy().into_owned();

    let mut s = store_at(store.path());
    assert_eq!(
        semlith::store::text_of(s.db(), &path).unwrap(),
        "",
        "indexed before acceptance"
    );
    s.accept(&path, "redacted", "cli").unwrap();
    s.index_paths(&[corpus.path().to_path_buf()], |_, _| {})
        .unwrap();
    let text = semlith::store::text_of(s.db(), &path).unwrap();
    assert!(!text.contains(&key), "the value reached the store: {text}");
    let line_two = text.lines().nth(1).unwrap_or_default();
    assert!(line_two.contains("[REDACTED:aws access key id]"), "{text}");
    // Accepted, so not refused: from 0.30.0 the row stays, carrying the
    // decision the Decisions tab lists, and is never an open refusal.
    assert!(
        semlith::store::refusal(s.db(), &path)
            .unwrap()
            .is_none_or(|r| r.accepted.is_some()),
        "an accepted file is still listed as refused"
    );

    // A comment edit keeps it indexed and redacted.
    fs::write(&file, format!("one, edited\nkey = \"{key}\"\nthree\n")).unwrap();
    s.index_paths(&[corpus.path().to_path_buf()], |_, _| {})
        .unwrap();
    let text = semlith::store::text_of(s.db(), &path).unwrap();
    assert!(text.contains("edited") && !text.contains(&key), "{text}");

    // A second key refuses it again and evicts it.
    let second = live("AWS");
    fs::write(
        &file,
        format!("one\nkey = \"{key}\"\nother = \"{second}\"\n"),
    )
    .unwrap();
    s.index_paths(&[corpus.path().to_path_buf()], |_, _| {})
        .unwrap();
    assert_eq!(
        semlith::store::text_of(s.db(), &path).unwrap(),
        "",
        "not evicted"
    );
    let row = semlith::store::refusal(s.db(), &path)
        .unwrap()
        .expect("listed again");
    assert!(row.rule.contains("new match since accepted"), "{row:?}");

    // As-is indexes the values, and revoke takes it out again.
    s.accept(&path, "as-is", "cli").unwrap();
    s.index_paths(&[corpus.path().to_path_buf()], |_, _| {})
        .unwrap();
    assert!(
        semlith::store::text_of(s.db(), &path)
            .unwrap()
            .contains(&second)
    );
    assert!(s.revoke(&path).unwrap());
    s.index_paths(&[corpus.path().to_path_buf()], |_, _| {})
        .unwrap();
    assert_eq!(semlith::store::text_of(s.db(), &path).unwrap(), "");
    assert!(semlith::store::refusal(s.db(), &path).unwrap().is_some());
}

/// A credential file is never accepted, whatever mode is asked for.
#[test]
fn a_credential_file_is_never_accepted() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    let env = write(
        corpus.path(),
        ".env",
        &format!("TOKEN={}\n", live("GitHub token")),
    );
    let rsa = write(corpus.path(), "id_rsa", &live("private key"));
    let mut s = store_at(store.path());
    for path in [&env, &rsa] {
        for mode in ["redacted", "as-is"] {
            let e = s
                .accept(&path.to_string_lossy(), mode, "cli")
                .expect_err("a credential file was accepted");
            assert!(e.to_string().contains("never accepted"), "{e}");
        }
    }
}

/// 2.7: with nothing changing between them, the plan's counts are the run's.
#[test]
#[ignore = "downloads an embedding model on first run"]
fn the_plan_counts_what_the_run_does() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    every_class(corpus.path());
    let mut s = store_at(store.path());
    let plan = s.plan(&[corpus.path().to_path_buf()]).unwrap();
    assert!(plan.seconds < 5.0, "{plan:?}");
    let report = s
        .index_paths(&[corpus.path().to_path_buf()], |_, _| {})
        .unwrap();
    assert_eq!(plan.embed, report.indexed, "{plan:?} {report:?}");
    assert_eq!(plan.unchanged, report.unchanged);
    let plan_not: usize = plan.not_indexed.values().sum();
    let rows = semlith::store::refusals(s.db()).unwrap();
    let run_not = rows.iter().filter(|r| r.class != "dummy").count();
    assert_eq!(plan_not, run_not, "{plan:?} {rows:#?}");
    // A second plan over the unchanged tree embeds nothing.
    let again = s.plan(&[corpus.path().to_path_buf()]).unwrap();
    assert_eq!(again.embed, 0, "{again:?}");
}

/// Nothing semlith writes or answers ever holds a full secret: not the
/// not-indexed list, not the ledger, not the store after a redacted
/// acceptance, not a read, and not a read from disk of the edited file.
#[test]
#[ignore = "downloads an embedding model on first run"]
fn no_output_ever_holds_a_full_secret() {
    let corpus = tempfile::tempdir().unwrap();
    let store = tempfile::tempdir().unwrap();
    let key = live("GitHub token");
    let file = write(
        corpus.path(),
        "ci.yml",
        &format!("env:\n  token: {key}\nsteps: []\n"),
    );
    run(store.path(), corpus.path(), false);
    let path = semlith::canonical(&file).to_string_lossy().into_owned();
    let mut s = store_at(store.path());

    let listed = serde_json::to_string(&semlith::store::refusals(s.db()).unwrap()).unwrap();
    assert!(!listed.contains(&key), "the list holds the key");
    assert!(listed.contains("ghp_…"), "the list shows no mask: {listed}");

    s.accept(&path, "redacted", "cli").unwrap();
    s.index_paths(&[corpus.path().to_path_buf()], |_, _| {})
        .unwrap();
    let everything: String = {
        let db = s.db();
        let mut stmt = db
            .prepare("SELECT COALESCE(group_concat(text, ''), '') FROM chunks")
            .unwrap();
        let chunks: String = stmt.query_row([], |r| r.get(0)).unwrap();
        let mut stmt = db
            .prepare("SELECT COALESCE(group_concat(query, ''), '') FROM retrievals")
            .unwrap();
        let ledger: String = stmt.query_row([], |r| r.get(0)).unwrap();
        let mut stmt = db
            .prepare("SELECT COALESCE(group_concat(fingerprints, ''), '') FROM acceptances")
            .unwrap();
        let prints: String = stmt.query_row([], |r| r.get(0)).unwrap();
        format!("{chunks}{ledger}{prints}")
    };
    assert!(!everything.contains(&key), "the store holds the key");
    let vector = s.embed_query("token").unwrap();
    let hits = s
        .search_preferring("token", &vector, 5, &Default::default(), Default::default())
        .unwrap();
    for (hit, _) in &hits {
        assert!(!hit.text.contains(&key), "a search returned the key");
    }
    let target = semlith::Target::parse(&format!("{path}:1-3"));
    let read = format!("{:?}", s.read(&target, &Default::default()).unwrap());
    assert!(!read.contains(&key), "a read returned the key");

    // Edited after indexing: the read comes from disk, and is redacted too.
    std::thread::sleep(std::time::Duration::from_millis(1100));
    let mut handle = fs::OpenOptions::new().append(true).open(&file).unwrap();
    writeln!(handle, "# edited").unwrap();
    drop(handle);
    let read = format!("{:?}", s.read(&target, &Default::default()).unwrap());
    assert!(
        !read.contains(&key),
        "a read from disk returned the key: {read}"
    );
    assert!(read.contains("REDACTED"), "{read}");
}