rete-cli 0.3.6

//! `rete export` with a filter, and `rete cost --dump` — issue #117 points 1
//! and 3 at the CLI surface.
//!
//! The filter is a **pruning** filter (the engine routes it to one permutation
//! and drops tiles a synopsis rejects, without fetching them), so the thing that
//! has to be tested is not that it is fast — the rete-core tests measure that —
//! but that it returns exactly the right rows. A pruning bug returns fewer rows
//! and no error, so every assertion here compares against the unfiltered export
//! filtered with `grep`-equivalent logic, never against a hand-written expected
//! list that could be wrong in the same direction.

mod common;

use predicates::prelude::*;

/// Three graphs, three predicates, literals and IRIs in the object column.
const QUADS: &str = concat!(
    "<http://ex/a> <http://ex/knows> <http://ex/b> .\n",
    "<http://ex/a> <http://ex/name> \"Alice\"@en .\n",
    "<http://ex/b> <http://ex/name> \"Bob\"@en .\n",
    "<http://ex/b> <http://ex/knows> <http://ex/a> .\n",
    "<http://ex/c> <http://ex/name> \"Carol\"@en <http://ex/g1> .\n",
    "<http://ex/c> <http://ex/knows> <http://ex/a> <http://ex/g1> .\n",
    "<http://ex/d> <http://ex/age> \"41\" <http://ex/g2> .\n",
);

fn built() -> (tempfile::TempDir, std::path::PathBuf) {
    let dir = tempfile::tempdir().unwrap();
    let src = dir.path().join("in.nq");
    let out = dir.path().join("out.rete");
    std::fs::write(&src, QUADS).unwrap();
    common::build(&src, &out, &["--no-pyramid"]);
    (dir, out)
}

fn export(file: &std::path::Path, args: &[&str]) -> Vec<String> {
    let out = common::rete()
        .arg("export")
        .arg(file)
        .args(args)
        .output()
        .unwrap();
    assert!(
        out.status.success(),
        "export {args:?} failed: {}",
        String::from_utf8_lossy(&out.stderr)
    );
    let mut lines: Vec<String> = String::from_utf8(out.stdout)
        .unwrap()
        .lines()
        .map(str::to_string)
        .filter(|l| !l.trim().is_empty())
        .collect();
    lines.sort();
    lines
}

/// Every filter shape must yield exactly the lines the unfiltered export yields
/// that contain that term — computed from the unfiltered output, so the two can
/// only agree by being right.
#[test]
fn a_filtered_export_is_the_full_export_filtered() {
    let (_dir, file) = built();
    let all = export(&file, &["--format", "nq"]);
    assert_eq!(all.len(), 7, "fixture changed: {all:#?}");

    // Predicate: bare IRI and `<iri>` token must behave identically.
    for token in ["http://ex/name", "<http://ex/name>"] {
        let got = export(&file, &["--format", "nq", "--predicate", token]);
        let want: Vec<String> = all
            .iter()
            .filter(|l| l.contains(" <http://ex/name> "))
            .cloned()
            .collect();
        assert_eq!(got, want, "--predicate {token}");
        assert_eq!(got.len(), 3);
    }

    // Subject — spans the default graph and a named one.
    let got = export(&file, &["--format", "nq", "--subject", "http://ex/c"]);
    assert_eq!(
        got,
        all.iter()
            .filter(|l| l.starts_with("<http://ex/c> "))
            .cloned()
            .collect::<Vec<_>>()
    );
    assert_eq!(got.len(), 2);

    // Object, as a language-tagged literal token (the shape a shell user has to
    // quote, and the one that would break if the CLI wrapped it in `<>`).
    let got = export(&file, &["--format", "nq", "--object", "\"Carol\"@en"]);
    assert_eq!(got.len(), 1);
    assert!(got[0].contains("\"Carol\"@en"), "{got:?}");

    // Two bound components at once.
    let got = export(
        &file,
        &[
            "--format",
            "nq",
            "--subject",
            "http://ex/b",
            "--predicate",
            "http://ex/knows",
        ],
    );
    assert_eq!(
        got,
        vec!["<http://ex/b> <http://ex/knows> <http://ex/a> .".to_string()]
    );

    // A term the dictionary does not contain matches nothing — and is not an
    // error, because "no such predicate" is a legitimate answer to a dump.
    assert!(export(&file, &["--format", "nq", "--predicate", "http://ex/nope"]).is_empty());
}

/// `--graph` selects one graph; absent means the default graph plus every named
/// one (the lossless N-Quads dump), and the empty string means the default
/// graph alone.
#[test]
fn graph_selection_slices_the_export() {
    let (_dir, file) = built();
    let all = export(&file, &["--format", "nq"]);

    let default_only = export(&file, &["--format", "nq", "--graph", ""]);
    assert_eq!(default_only.len(), 4);
    assert!(
        default_only
            .iter()
            .all(|l| !l.contains("<http://ex/g1>") && !l.contains("<http://ex/g2>")),
        "the default graph must not carry a graph token: {default_only:?}"
    );

    for token in ["http://ex/g1", "<http://ex/g1>"] {
        let g1 = export(&file, &["--format", "nq", "--graph", token]);
        assert_eq!(g1.len(), 2, "--graph {token}");
        assert!(g1.iter().all(|l| l.contains("<http://ex/g1> .")));
    }

    // The parts partition the whole.
    let g2 = export(&file, &["--format", "nq", "--graph", "http://ex/g2"]);
    assert_eq!(default_only.len() + 2 + g2.len(), all.len());

    // Filters compose with the graph.
    let got = export(
        &file,
        &[
            "--format",
            "nq",
            "--graph",
            "http://ex/g1",
            "--predicate",
            "http://ex/knows",
        ],
    );
    assert_eq!(got.len(), 1);
    assert!(got[0].contains("<http://ex/g1>"));

    // Turtle/JSON-LD are single-graph, and now that graph can be a named one.
    common::rete()
        .arg("export")
        .arg(&file)
        .args(["--format", "ttl", "--graph", "http://ex/g1"])
        .assert()
        .success()
        .stdout(predicate::str::contains("\"Carol\"@en"));
}

/// `rete cost --dump` previews a dump the way `rete cost <query>` previews a
/// query: same `AccessCost` line shape, same JSON envelope conventions.
///
/// The index figure is *computed from the tile directories*, so the assertions
/// worth making are the ones a sampled estimate could not satisfy: a filter
/// admits no more tiles than routing leaves, routing leaves no more than the
/// section has, and a filter term the file does not contain plans no scan at
/// all rather than a scan of everything.
#[test]
fn dump_cost_previews_a_dump_without_running_it() {
    let (_dir, file) = built();

    let full: serde_json::Value = common::json(
        common::rete()
            .arg("cost")
            .arg(&file)
            .args(["--dump", "--json"]),
    );
    assert_eq!(full["mode"], "dump");
    assert_eq!(full["source_kind"], "local");
    assert_eq!(full["lazy_dump_open"]["available"], true);
    assert!(full["lazy_dump_open"]["bytes"].as_u64().unwrap() > 0);
    // Default graph + two named graphs.
    assert_eq!(full["graphs"].as_array().unwrap().len(), 3);
    let full_tiles = full["index_tiles"]["bytes"].as_u64().unwrap();
    assert!(full_tiles > 0);
    // The floor is the open plus the admitted tiles; the ceiling adds the
    // dictionary. Both are stated, neither is guessed.
    let floor = full["estimated_bytes"]["floor"].as_u64().unwrap();
    let ceiling = full["estimated_bytes"]["ceiling"].as_u64().unwrap();
    assert_eq!(
        ceiling - floor,
        full["dictionary_ceiling_bytes"].as_u64().unwrap()
    );
    // No `floor <= file_bytes` assertion: on a fixture this small the lazy
    // open's block-aligned reads and its chunk-directory probe over-read the
    // whole file several times, so the floor exceeds the file length. That is
    // the same small-file artefact `cost_cli` documents for the query preview,
    // and pinning it here would pin the artefact rather than the contract.
    assert!(floor > 0);

    // One graph is a smaller preview than every graph.
    let one: serde_json::Value = common::json(common::rete().arg("cost").arg(&file).args([
        "--dump",
        "--graph",
        "http://ex/g1",
        "--json",
    ]));
    assert_eq!(one["graphs"].as_array().unwrap().len(), 1);
    assert!(one["index_tiles"]["bytes"].as_u64().unwrap() < full_tiles);

    // Every planned scan routes inside {SPO, POS, OSP} — the three permutations
    // a `--permutations 3` file is guaranteed to carry.
    let pred: serde_json::Value = common::json(common::rete().arg("cost").arg(&file).args([
        "--dump",
        "--predicate",
        "http://ex/name",
        "--json",
    ]));
    for g in pred["graphs"].as_array().unwrap() {
        if g["matches"].as_bool() == Some(true) {
            let perm = g["permutation"].as_str().unwrap();
            assert!(["SPO", "POS", "OSP"].contains(&perm), "routed to {perm}");
            assert!(g["tiles_admitted"].as_u64().unwrap() <= g["tiles_routed"].as_u64().unwrap());
            assert!(g["tiles_routed"].as_u64().unwrap() <= g["tiles_total"].as_u64().unwrap());
        }
    }

    // A term the file does not contain: no scan planned, zero index bytes.
    let miss: serde_json::Value = common::json(common::rete().arg("cost").arg(&file).args([
        "--dump",
        "--predicate",
        "http://ex/nope",
        "--json",
    ]));
    assert_eq!(miss["index_tiles"]["bytes"], 0);
    assert!(miss["graphs"]
        .as_array()
        .unwrap()
        .iter()
        .all(|g| g["matches"] == false));

    // The human-readable form says the same thing, and says what it cannot know.
    common::rete()
        .arg("cost")
        .arg(&file)
        .args(["--dump", "--predicate", "http://ex/name"])
        .assert()
        .success()
        .stdout(predicate::str::contains("dump cost preview"))
        .stdout(predicate::str::contains("no tile fetched"))
        .stdout(predicate::str::contains("dictionary ceiling"));

    // `rete cost` with neither a query nor --dump is an error, not a panic.
    common::rete()
        .arg("cost")
        .arg(&file)
        .assert()
        .failure()
        .stderr(predicate::str::contains("--dump"));
}

/// The reader choice must not change the bytes. `rete export` streams through
/// the lazy ranged reader by default; `--in-memory` forces the old whole-file
/// load. Both must emit the identical dump (same order, same bytes) for every
/// slice — the flag trades RAM for speed, never output. Guards the invariant the
/// RSS win rests on: if these ever diverge, the default silently changed data.
#[test]
fn in_memory_and_ranged_dumps_are_byte_identical() {
    let (_dir, file) = built();
    let raw = |args: &[&str]| -> Vec<u8> {
        let out = common::rete()
            .arg("export")
            .arg(&file)
            .args(args)
            .output()
            .unwrap();
        assert!(out.status.success(), "export {args:?} failed");
        out.stdout
    };
    for slice in [
        &["--format", "nq"][..],
        &["--format", "nq", "--graph", ""][..],
        &["--format", "nq", "--graph", "http://ex/g1"][..],
        &["--format", "nq", "--predicate", "http://ex/knows"][..],
        &["--format", "ttl"][..],
        &["--format", "jsonld"][..],
    ] {
        let ranged = raw(slice);
        let mut in_mem_args = slice.to_vec();
        in_mem_args.push("--in-memory");
        let in_memory = raw(&in_mem_args);
        assert_eq!(
            ranged, in_memory,
            "ranged vs --in-memory dump differs for {slice:?}"
        );
    }
}

/// The same byte-identity guarantee, but on a dictionary that spans **many
/// chunks** — the case the chunk-ordered windowed resolver (bounded-export
/// phase 1) actually restructures. The small `QUADS` fixture fits every section
/// in one chunk, so it never crosses a chunk boundary within a window; this
/// builds a file whose subject and object sections each run to several ~64 KiB
/// chunks (thousands of distinct terms, plus one multi-KiB literal that forces
/// a run/chunk boundary around a large term), then asserts the ranged
/// (windowed) dump is byte-for-byte the eager `--in-memory` dump — same order,
/// same bytes — for the full dump and a predicate slice. If the window's
/// chunk-order grouping or its scan-order emission ever changed a byte, these
/// diverge.
#[test]
fn windowed_multichunk_dump_matches_eager() {
    let dir = tempfile::tempdir().unwrap();
    let src = dir.path().join("big.nt");
    let out = dir.path().join("big.rete");
    let mut nt = String::new();
    // 4000 triples with distinct subjects and distinct ~70-byte object literals
    // -> the subject and object dictionary sections each exceed the 64 KiB
    // chunk budget and are cut into several chunks.
    for i in 0..4000u32 {
        nt.push_str(&format!(
            "<http://example.org/subject/{i:07}> <http://example.org/p/{}> \
             \"object literal number {i:07} with padding text to widen the term\" .\n",
            i % 8
        ));
    }
    // A multi-KiB literal forces a run/chunk boundary around a large term, the
    // window edge case the reorder must not disturb.
    nt.push_str(&format!(
        "<http://example.org/subject/big> <http://example.org/p/big> \"{}\" .\n",
        "Z".repeat(8000)
    ));
    std::fs::write(&src, &nt).unwrap();
    common::build(&src, &out, &["--no-pyramid"]);

    let raw = |args: &[&str]| -> Vec<u8> {
        let o = common::rete()
            .arg("export")
            .arg(&out)
            .args(args)
            .output()
            .unwrap();
        assert!(
            o.status.success(),
            "export {args:?} failed: {}",
            String::from_utf8_lossy(&o.stderr)
        );
        o.stdout
    };
    for slice in [
        &["--format", "nq"][..],
        &["--format", "nq", "--predicate", "http://example.org/p/3"][..],
    ] {
        let ranged = raw(slice);
        assert!(!ranged.is_empty(), "expected rows for {slice:?}");
        let mut mem = slice.to_vec();
        mem.push("--in-memory");
        let in_memory = raw(&mem);
        assert_eq!(
            ranged, in_memory,
            "windowed multi-chunk dump differs for {slice:?}"
        );
        // Bounded-export phase 2: a small `--memory-budget-mb` must emit the
        // identical bytes — the budget bounds memory, never output.
        let mut budgeted_args = slice.to_vec();
        budgeted_args.extend_from_slice(&["--memory-budget-mb", "64"]);
        let budgeted = raw(&budgeted_args);
        assert_eq!(
            ranged, budgeted,
            "--memory-budget-mb 64 dump differs from default for {slice:?}"
        );
    }
}

/// **Bounded-export phase 2 gate:** `--memory-budget-mb 64` yields byte-for-byte
/// the `--in-memory` dump, for the full dump and a filtered slice. The budget
/// caps the evictable dictionary/tile caches; it must change memory, not bytes.
#[test]
fn memory_budget_dump_matches_in_memory() {
    let (_dir, file) = built();
    let raw = |args: &[&str]| -> Vec<u8> {
        let out = common::rete()
            .arg("export")
            .arg(&file)
            .args(args)
            .output()
            .unwrap();
        assert!(
            out.status.success(),
            "export {args:?} failed: {}",
            String::from_utf8_lossy(&out.stderr)
        );
        out.stdout
    };
    for slice in [
        &["--format", "nq"][..],
        &["--format", "nq", "--predicate", "http://ex/knows"][..],
    ] {
        let mut budgeted = slice.to_vec();
        budgeted.extend_from_slice(&["--memory-budget-mb", "64"]);
        let mut in_mem = slice.to_vec();
        in_mem.push("--in-memory");
        assert_eq!(
            raw(&budgeted),
            raw(&in_mem),
            "--memory-budget-mb 64 vs --in-memory differ for {slice:?}"
        );
    }
}

/// `RETE_MEM_TRACE` is a diagnostic only: it brackets the dump with `open` and
/// `end` lines naming the cache residency and the fixed directories the budget
/// charges, on stderr, and leaves stdout byte-identical.
#[test]
fn mem_trace_brackets_the_dump_and_leaves_output_unchanged() {
    let (_dir, file) = built();
    let plain = common::rete()
        .arg("export")
        .arg(&file)
        .args(["--memory-budget-mb", "64"])
        .output()
        .unwrap();
    let traced = common::rete()
        .arg("export")
        .arg(&file)
        .args(["--memory-budget-mb", "64"])
        .env("RETE_MEM_TRACE", "0")
        .output()
        .unwrap();
    assert!(plain.status.success() && traced.status.success());
    assert_eq!(plain.stdout, traced.stdout, "the trace changed the dump");
    let err = String::from_utf8_lossy(&traced.stderr);
    for slot in ["slot=open", "slot=end"] {
        let line = err
            .lines()
            .find(|l| l.starts_with("[memtrace]") && l.contains(slot))
            .unwrap_or_else(|| panic!("no {slot} line in:\n{err}"));
        for field in [
            "rss_mb=",
            "hwm_mb=",
            "dict_res_mb=",
            "fixed_mb=",
            "dict_dir_mb=",
        ] {
            assert!(line.contains(field), "{slot} line lacks {field}: {line}");
        }
    }
    assert!(
        !String::from_utf8_lossy(&plain.stderr).contains("[memtrace]"),
        "the trace must be off unless RETE_MEM_TRACE is set"
    );
}