graphitesql 0.1.6

//! Roadmap D2e-encoder: graphite-written FTS5 stays BYTE-IDENTICAL to sqlite for
//! MULTI-TERM segments — the leaf-fill / term-split boundary.
//!
//! A segment holding many distinct terms is paginated across leaf pages. sqlite
//! ends the current leaf and starts the next based purely on the page header, the
//! committed page-index footer, and the FULL (uncompressed) length of the term
//! about to be written (`fts5WriteAppendTerm`: `buf.n + pgidx.n + nTerm + 2 >=
//! pgsz`) — the term's doclist size does NOT enter that decision. graphite used a
//! coarser estimate that folded in the doclist and used the compressed record
//! length, so it split a leaf a few bytes early and every downstream leaf's bytes
//! (and the `%_idx` separators) drifted from sqlite's. This test pins the ported
//! boundary: for a range of multi-term corpora the raw `%_data`/`%_idx` bytes are
//! identical to sqlite's own single-segment (optimized) index.
//!
//! Each case builds the SAME corpus with graphite and with stock `sqlite3`
//! (3.50.4, FTS5) + `optimize` (so sqlite compacts to one segment like graphite's
//! always-one-segment rebuild), then asserts: byte-identical `%_data` and `%_idx`,
//! sqlite's `integrity-check` accepts graphite's file, and a `MATCH` returns the
//! same rows. Skipped when `sqlite3` with FTS5 is not on PATH.

#![cfg(feature = "std")]
#![cfg(feature = "fts5")]

use graphitesql::Connection;
use std::io::Write;
use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};

fn tmp_path(tag: &str) -> String {
    static SEQ: AtomicU64 = AtomicU64::new(0);
    let p = std::env::temp_dir().join(format!(
        "gsql-fts5-mt-{}-{}-{}.db",
        tag,
        std::process::id(),
        SEQ.fetch_add(1, Ordering::Relaxed)
    ));
    let p = p.to_string_lossy().into_owned();
    let _ = std::fs::remove_file(&p);
    p
}

/// `sqlite3` with FTS5 available on PATH?
fn have_fts5_sqlite() -> bool {
    let o = Command::new("sqlite3")
        .arg(":memory:")
        .arg("CREATE VIRTUAL TABLE t USING fts5(a); SELECT 1;")
        .output();
    matches!(o, Ok(o) if o.status.success())
}

/// Run `q` through stock sqlite3 on `path`; assert success; return raw stdout.
fn sqlite_raw(path: &str, q: &str) -> String {
    let o = Command::new("sqlite3").arg(path).arg(q).output().unwrap();
    assert!(
        o.status.success(),
        "sqlite3 failed for {q:?}: {}",
        String::from_utf8_lossy(&o.stderr)
    );
    String::from_utf8_lossy(&o.stdout).trim_end().to_string()
}

/// Run `sql` through sqlite3 via stdin (avoids ARG_MAX for large VALUES lists).
fn sqlite_stdin(path: &str, sql: &str) {
    let mut child = Command::new("sqlite3")
        .arg(path)
        .stdin(Stdio::piped())
        .stdout(Stdio::null())
        .spawn()
        .unwrap();
    child
        .stdin
        .take()
        .unwrap()
        .write_all(sql.as_bytes())
        .unwrap();
    assert!(child.wait().unwrap().success());
}

/// sqlite's FTS5 `integrity-check` must accept the file (no error).
fn sqlite_integrity_ok(path: &str) -> bool {
    Command::new("sqlite3")
        .arg(path)
        .arg("INSERT INTO ft(ft) VALUES('integrity-check');")
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

/// Build the same corpus with graphite (`g`) and with sqlite+optimize (`s`).
fn build_pair(g: &str, s: &str, values: &str) {
    {
        let mut c = Connection::create(g).unwrap();
        c.execute("CREATE VIRTUAL TABLE ft USING fts5(body)")
            .unwrap();
        // One statement → one bulk-rebuild (graphite writes a single segment).
        c.execute("BEGIN").ok();
        c.execute(&format!("INSERT INTO ft(rowid,body) VALUES {values}"))
            .unwrap();
        c.execute("COMMIT").ok();
    }
    sqlite_stdin(
        s,
        &format!(
            "CREATE VIRTUAL TABLE ft USING fts5(body);\n\
             INSERT INTO ft(rowid,body) VALUES {values};\n\
             INSERT INTO ft(ft) VALUES('optimize');"
        ),
    );
}

/// A corpus of `n_docs` documents, each `terms_per_doc` space-separated tokens
/// drawn from a `vocab`-sized pool of fixed-width `termNNNN` words. Chosen so the
/// segment holds MANY distinct terms and paginates across several leaves.
fn multiterm_values(n_docs: usize, terms_per_doc: usize, vocab: usize) -> String {
    let mut s = String::new();
    for d in 1..=n_docs {
        if d > 1 {
            s.push(',');
        }
        let mut body = String::new();
        for w in 0..terms_per_doc {
            if w > 0 {
                body.push(' ');
            }
            let t = (d * 7 + w * 13) % vocab;
            body.push_str(&format!("term{t:04}"));
        }
        s.push_str(&format!("({d},'{body}')"));
    }
    s
}

/// A corpus with VARIABLE-length terms (1..=9 leading `z`s + a decimal index),
/// so prefix-compression and the full-term-length term-split interact.
fn varlen_values(n_docs: usize, terms_per_doc: usize, vocab: usize) -> String {
    let mut s = String::new();
    for d in 1..=n_docs {
        if d > 1 {
            s.push(',');
        }
        let mut body = String::new();
        for w in 0..terms_per_doc {
            if w > 0 {
                body.push(' ');
            }
            let t = (d * 7 + w * 13) % vocab;
            body.push_str(&"z".repeat(1 + (t % 9)));
            body.push_str(&format!("{t}"));
        }
        s.push_str(&format!("({d},'{body}')"));
    }
    s
}

/// Assert graphite's `%_data` + `%_idx` are byte-identical to sqlite's, that
/// sqlite's integrity-check accepts graphite's file, and that a `MATCH` on a
/// known term returns identical rows through both engines.
fn assert_byte_identical(g: &str, s: &str, min_leaves: usize) {
    let n_leaves: usize = sqlite_raw(g, "SELECT count(*) FROM ft_data WHERE id>10;")
        .parse()
        .unwrap();
    assert!(
        n_leaves >= min_leaves,
        "corpus produced only {n_leaves} leaves; expected >= {min_leaves}"
    );

    assert!(
        sqlite_integrity_ok(g),
        "sqlite integrity-check rejected graphite's multi-term file"
    );
    assert_eq!(sqlite_raw(g, "PRAGMA integrity_check;"), "ok");

    let data = "SELECT id||':'||quote(block) FROM ft_data ORDER BY id;";
    assert_eq!(
        sqlite_raw(g, data),
        sqlite_raw(s, data),
        "%_data bytes diverge from sqlite ({n_leaves} leaves)"
    );
    let idx = "SELECT segid||':'||quote(term)||':'||pgno FROM ft_idx ORDER BY segid, term;";
    assert_eq!(
        sqlite_raw(g, idx),
        sqlite_raw(s, idx),
        "%_idx bytes diverge from sqlite ({n_leaves} leaves)"
    );

    // MATCH agreement on a term that is present in the corpus.
    let m = "SELECT count(*) FROM ft WHERE ft MATCH 'term0000';";
    assert_eq!(sqlite_raw(g, m), sqlite_raw(s, m), "MATCH count diverges");
}

#[test]
fn multiterm_leaf_fill_is_byte_identical() {
    if !have_fts5_sqlite() {
        eprintln!("sqlite3 with FTS5 not found; skipping");
        return;
    }
    // (n_docs, terms_per_doc, vocab, min_expected_leaves)
    for &(nd, tpd, vocab, min_leaves) in &[
        (200usize, 10usize, 400usize, 2usize),
        (500, 12, 800, 5),
        (1000, 15, 1500, 10),
        (3000, 8, 6000, 20),
    ] {
        let g = tmp_path("mt-g");
        let s = tmp_path("mt-s");
        build_pair(&g, &s, &multiterm_values(nd, tpd, vocab));
        assert_byte_identical(&g, &s, min_leaves);
        let _ = std::fs::remove_file(&g);
        let _ = std::fs::remove_file(&s);
    }
}

#[test]
fn multiterm_varlen_leaf_fill_is_byte_identical() {
    if !have_fts5_sqlite() {
        eprintln!("sqlite3 with FTS5 not found; skipping");
        return;
    }
    for &(nd, tpd, vocab, min_leaves) in &[
        (300usize, 15usize, 600usize, 3usize),
        (800, 20, 1500, 10),
        (1500, 10, 3000, 15),
    ] {
        let g = tmp_path("vl-g");
        let s = tmp_path("vl-s");
        // varlen corpora don't contain 'term0000'; reuse the byte checks but skip
        // the MATCH-on-term0000 assertion by checking a present term instead.
        {
            let mut c = Connection::create(&g).unwrap();
            c.execute("CREATE VIRTUAL TABLE ft USING fts5(body)")
                .unwrap();
            c.execute("BEGIN").ok();
            c.execute(&format!(
                "INSERT INTO ft(rowid,body) VALUES {}",
                varlen_values(nd, tpd, vocab)
            ))
            .unwrap();
            c.execute("COMMIT").ok();
        }
        sqlite_stdin(
            &s,
            &format!(
                "CREATE VIRTUAL TABLE ft USING fts5(body);\n\
                 INSERT INTO ft(rowid,body) VALUES {};\n\
                 INSERT INTO ft(ft) VALUES('optimize');",
                varlen_values(nd, tpd, vocab)
            ),
        );

        let n_leaves: usize = sqlite_raw(&g, "SELECT count(*) FROM ft_data WHERE id>10;")
            .parse()
            .unwrap();
        assert!(
            n_leaves >= min_leaves,
            "varlen corpus produced only {n_leaves} leaves; expected >= {min_leaves}"
        );
        assert!(sqlite_integrity_ok(&g));
        assert_eq!(sqlite_raw(&g, "PRAGMA integrity_check;"), "ok");
        let data = "SELECT id||':'||quote(block) FROM ft_data ORDER BY id;";
        assert_eq!(
            sqlite_raw(&g, data),
            sqlite_raw(&s, data),
            "varlen %_data bytes diverge ({n_leaves} leaves)"
        );
        let idx = "SELECT segid||':'||quote(term)||':'||pgno FROM ft_idx ORDER BY segid, term;";
        assert_eq!(
            sqlite_raw(&g, idx),
            sqlite_raw(&s, idx),
            "varlen %_idx bytes diverge ({n_leaves} leaves)"
        );
        let _ = std::fs::remove_file(&g);
        let _ = std::fs::remove_file(&s);
    }
}