hayson-engine 0.27.0

//! A database opened with `encryption` keys keeps nothing readable on disk, and still behaves like
//! any other: indexes, unique constraints, images, units of work, compaction, reopen.

use std::fs;
use std::path::Path;

use hayson_engine::crypto::{CipherOptions, PasswordHashParams};
use hayson_engine::database::{Database, DatabaseError, OpenOptions, RestoreOutcome};
use hayson_engine::query::{ComparisonOp, QueryBuilder};
use serde_json::json;
use tempfile::TempDir;

fn keys(byte: u8) -> CipherOptions {
    CipherOptions {
        keys: [("k1".to_string(), vec![byte; 32])].into(),
        key_id: Some("k1".to_string()),
        on_missing_key: None,
    }
}

fn open_with(dir: &Path, encryption: Option<CipherOptions>) -> Result<Database, DatabaseError> {
    Database::open_with(
        dir,
        OpenOptions {
            encryption,
            password_hash: PasswordHashParams {
                memory_kib: 64,
                iterations: 1,
                parallelism: 1,
            },
            ..OpenOptions::default()
        },
    )
}

fn people_schema() -> serde_json::Value {
    json!({
        "schemaVersion": 1,
        "table": "people",
        "primaryKey": "id",
        "fields": {
            "id": { "type": "string" },
            "email": { "type": "string", "unique": true },
            "name": { "type": "string", "index": "exact" },
            "photo": { "type": "image" },
            "password": { "type": "string", "passwordHash": true },
        },
    })
}

fn notes_schema() -> serde_json::Value {
    json!({
        "schemaVersion": 1,
        "table": "notes",
        "primaryKey": "id",
        "partitionBy": "kind",
        "fields": { "id": { "type": "string" }, "kind": { "type": "string", "required": true }, "text": { "type": "string" } },
    })
}

/// Every regular file under `dir`, with its bytes.
fn files_under(dir: &Path) -> Vec<(String, Vec<u8>)> {
    let mut found = Vec::new();
    let mut stack = vec![dir.to_path_buf()];
    while let Some(next) = stack.pop() {
        for entry in fs::read_dir(&next).unwrap().filter_map(Result::ok) {
            let path = entry.path();
            if path.is_dir() {
                stack.push(path);
            } else if !path.to_string_lossy().ends_with(".lock") {
                found.push((path.display().to_string(), fs::read(&path).unwrap()));
            }
        }
    }
    found
}

fn contains(haystack: &[u8], needle: &str) -> bool {
    haystack.windows(needle.len()).any(|window| window == needle.as_bytes())
}

fn seed(dir: &Path) {
    let mut db = open_with(dir, Some(keys(7))).unwrap();
    db.create_table("people", people_schema()).unwrap();
    db.create_table("notes", notes_schema()).unwrap();
    let people = db.table_mut("people").unwrap();
    people
        .put(
            &json!({
                "id": "p1", "email": "ann@example.test", "name": "Ann Kowalska",
                "photo": { "content": "c2VjcmV0LXBob3Rv", "virtualPath": "/ann.png" }, // "secret-photo"
                "password": "correct horse battery",
            }),
            false,
        )
        .unwrap();
    people
        .put(
            &json!({ "id": "p2", "email": "bob@example.test", "name": "Bob Nowak" }),
            false,
        )
        .unwrap();
    db.table_mut("notes")
        .unwrap()
        .put(
            &json!({ "id": "n1", "kind": "diary", "text": "dear diary, my secret is x" }),
            false,
        )
        .unwrap();
    db.close().unwrap();
}

#[test]
fn nothing_readable_reaches_the_disk_and_the_database_reads_back_whole() {
    let dir = TempDir::new().unwrap();
    seed(dir.path());

    for (path, bytes) in files_under(dir.path()) {
        for secret in [
            "Ann",
            "Kowalska",
            "ann@example",
            "secret-photo",
            "dear diary",
            "Nowak",
            "people",
            "email",
        ] {
            // File names may name the table; the contents never do.
            assert!(!contains(&bytes, secret), "{path} leaks {secret:?}");
        }
    }
    let log = fs::read_to_string(dir.path().join("people.jsonl")).unwrap();
    assert!(log.lines().all(|line| line.starts_with("$hayson-seal$v1$k1$")), "{log}");
    assert!(fs::read(dir.path().join("hayson.json")).unwrap().starts_with(b"HSEAL"));

    let db = open_with(dir.path(), Some(keys(7))).unwrap();
    assert_eq!(
        db.restore_outcome("people"),
        Some(RestoreOutcome::Restored),
        "the sealed snapshot was used"
    );
    let people = db.table("people").unwrap();
    assert_eq!(people.get("p1").unwrap().unwrap()["name"], json!("Ann Kowalska"));
    assert_eq!(people.keys().count(), 2);
    assert!(
        people
            .verify_password("p1", "password", "correct horse battery")
            .unwrap()
            .ok
    );
    let images = people.list_images().unwrap();
    assert_eq!(images.len(), 1);
    assert_eq!(
        images[0].bytes,
        "secret-photo".len() as u64,
        "reported as the plain size"
    );
    assert_eq!(
        people.read_image(&images[0].physical_path).unwrap(),
        Some(b"secret-photo".to_vec())
    );
    assert_eq!(
        db.table("notes").unwrap().get("n1").unwrap().unwrap()["kind"],
        json!("diary")
    );
}

#[test]
fn unique_constraints_and_indexes_still_work_over_encrypted_logs() {
    let dir = TempDir::new().unwrap();
    seed(dir.path());
    let mut db = open_with(dir.path(), Some(keys(7))).unwrap();
    let people = db.table_mut("people").unwrap();

    let clash = people.put(
        &json!({ "id": "p3", "email": "ann@example.test", "name": "Other" }),
        false,
    );
    assert!(clash.is_err(), "the unique email was not enforced");
    let spec = QueryBuilder::new()
        .r#where("name", ComparisonOp::Eq, json!("Bob Nowak"))
        .spec();
    let found = people.query(&spec).unwrap();
    assert_eq!(found.keys, vec!["p2".to_string()]);
}

#[test]
fn the_wrong_key_no_key_and_a_key_on_a_plain_database_are_refused_cleanly() {
    let dir = TempDir::new().unwrap();
    seed(dir.path());

    let wrong = open_with(dir.path(), Some(keys(8))).err().expect("wrong key opened it");
    let missing = open_with(dir.path(), None).err().expect("no key opened it");
    assert!(matches!(wrong, DatabaseError::Cipher(_)), "{wrong:?}");
    assert!(matches!(missing, DatabaseError::Cipher(_)), "{missing:?}");
    assert!(wrong.to_string().contains("authentication"), "{wrong}");
    assert!(missing.to_string().contains("encrypted"), "{missing}");

    let plain = TempDir::new().unwrap();
    let mut db = open_with(plain.path(), None).unwrap();
    db.create_table("notes", notes_schema()).unwrap();
    db.close().unwrap();
    let keyed = open_with(plain.path(), Some(keys(7)))
        .err()
        .expect("a plain database opened with a key");
    assert!(matches!(keyed, DatabaseError::Cipher(_)), "{keyed:?}");
    assert!(keyed.to_string().contains("not encrypted"), "{keyed}");
}

#[test]
fn compaction_reseals_the_log_and_the_data_survives_a_reopen() {
    let dir = TempDir::new().unwrap();
    seed(dir.path());
    let mut db = open_with(dir.path(), Some(keys(7))).unwrap();
    let people = db.table_mut("people").unwrap();
    for round in 0..5 {
        people
            .patch("p2", &json!({ "name": format!("Bob {round}") }), false)
            .unwrap();
    }
    let before = fs::metadata(dir.path().join("people.jsonl")).unwrap().len();
    db.table_mut("people").unwrap().compact().unwrap();
    db.close().unwrap();
    assert!(fs::metadata(dir.path().join("people.jsonl")).unwrap().len() < before);

    let db = open_with(dir.path(), Some(keys(7))).unwrap();
    assert_eq!(
        db.table("people").unwrap().get("p2").unwrap().unwrap()["name"],
        json!("Bob 4")
    );
    let log = fs::read_to_string(dir.path().join("people.jsonl")).unwrap();
    assert!(log.lines().all(|line| line.starts_with("$hayson-seal$")));
}

#[test]
fn a_tampered_line_is_a_corrupt_log_with_its_offset_and_a_half_written_tail_is_cut_off() {
    let dir = TempDir::new().unwrap();
    seed(dir.path());
    let log_path = dir.path().join("people.jsonl");
    let original = fs::read_to_string(&log_path).unwrap();
    let first = original.lines().next().unwrap();
    let second_offset = first.len() + 1;

    // Flip one character in the second line's ciphertext.
    let mut tampered = original.clone().into_bytes();
    let at = second_offset + original.lines().nth(1).unwrap().len() - 3;
    tampered[at] = if tampered[at] == b'A' { b'B' } else { b'A' };
    fs::write(&log_path, &tampered).unwrap();
    let _ = fs::remove_file(dir.path().join("people.idx"));
    let _ = fs::remove_dir_all(dir.path().join("index"));
    for entry in fs::read_dir(dir.path()).unwrap().filter_map(Result::ok) {
        if entry.path().extension().is_some_and(|e| e == "idx") {
            fs::remove_file(entry.path()).unwrap();
        }
    }
    let error = open_with(dir.path(), Some(keys(7)))
        .err()
        .expect("a tampered log opened");
    let text = error.to_string();
    assert!(
        text.contains("corrupt") && text.contains(&format!("offset {second_offset}")),
        "{text}"
    );

    // A tail without its newline is what a killed writer leaves; it is cut off, the rest stays.
    fs::write(&log_path, format!("{original}$hayson-seal$v1$k1$AAAA")).unwrap();
    let db = open_with(dir.path(), Some(keys(7))).unwrap();
    assert_eq!(db.table("people").unwrap().keys().count(), 2);
    db.close().unwrap();
    assert_eq!(fs::read_to_string(&log_path).unwrap(), original);
}

#[test]
fn a_unit_of_work_across_tables_lands_sealed() {
    let dir = TempDir::new().unwrap();
    seed(dir.path());
    let mut db = open_with(dir.path(), Some(keys(7))).unwrap();
    let mut unit = db.unit_of_work();
    unit.put(
        "people",
        json!({ "id": "p9", "email": "zed@example.test", "name": "Zed" }),
    );
    unit.put(
        "notes",
        json!({ "id": "n2", "kind": "work", "text": "quarterly numbers" }),
    );

    db.commit(&mut unit).unwrap();
    db.close().unwrap();

    for (path, bytes) in files_under(dir.path()) {
        assert!(!contains(&bytes, "Zed") && !contains(&bytes, "quarterly"), "{path}");
    }
    assert!(
        !dir.path().join(".hayson.txn").exists(),
        "the journal is gone once the unit landed"
    );
    let db = open_with(dir.path(), Some(keys(7))).unwrap();
    assert!(db.table("notes").unwrap().get("n2").unwrap().is_some());
}

#[test]
fn the_nested_layout_encrypts_too() {
    use hayson_engine::database::{LayoutKind, LayoutOptions};
    let dir = TempDir::new().unwrap();
    let open_nested = |key: u8| {
        Database::open_with(
            dir.path(),
            OpenOptions {
                encryption: Some(keys(key)),
                layout: LayoutOptions {
                    kind: Some(LayoutKind::Nested),
                    ..LayoutOptions::default()
                },
                ..OpenOptions::default()
            },
        )
    };
    let mut db = open_nested(7).unwrap();
    db.create_table("notes", notes_schema()).unwrap();
    db.table_mut("notes")
        .unwrap()
        .put(&json!({ "id": "n1", "kind": "diary", "text": "nested secret" }), false)
        .unwrap();
    db.close().unwrap();

    for (path, bytes) in files_under(dir.path()) {
        assert!(!contains(&bytes, "nested secret"), "{path}");
    }
    // Reopened without saying "nested": the layout is read out of the sealed catalog.
    let db = Database::open_with(
        dir.path(),
        OpenOptions {
            encryption: Some(keys(7)),
            ..OpenOptions::default()
        },
    )
    .unwrap();
    assert!(db.table("notes").unwrap().get("n1").unwrap().is_some());
}

fn parked(dir: &Path) -> TempDir {
    let _ = dir;
    TempDir::new().unwrap()
}

#[test]
fn verify_needs_the_key_and_then_checks_everything_a_plain_verify_would() {
    use hayson_engine::database::{verify, VerifyOptions};
    let dir = TempDir::new().unwrap();
    seed(dir.path());

    let without = verify(dir.path(), &VerifyOptions::default());
    assert!(
        without.problems.iter().any(|p| p.code == "catalog-encrypted"),
        "{:?}",
        without.problems
    );

    let wrong = verify(
        dir.path(),
        &VerifyOptions {
            encryption: Some(keys(8)),
            ..VerifyOptions::default()
        },
    );
    assert!(
        wrong.problems.iter().any(|p| p.code == "catalog-key-rejected"),
        "{:?}",
        wrong.problems
    );

    let with = VerifyOptions {
        encryption: Some(keys(7)),
        ..VerifyOptions::default()
    };
    let clean = verify(dir.path(), &with);
    assert!(clean.problems.is_empty(), "{:?}", clean.problems);
    assert!(clean.log_lines_checked >= 3);

    // A damaged line is found by offset, an image whose bytes changed is found by its seal.
    let log = dir.path().join("people.jsonl");
    let text = fs::read_to_string(&log).unwrap();
    let mut bytes = text.clone().into_bytes();
    let at = text.lines().next().unwrap().len() - 4;
    bytes[at] = if bytes[at] == b'A' { b'B' } else { b'A' };
    fs::write(&log, bytes).unwrap();
    let broken = verify(dir.path(), &with);
    assert!(
        broken
            .problems
            .iter()
            .any(|p| p.code == "log-corrupt-line" && p.offset == Some(0)),
        "{:?}",
        broken.problems
    );
}

#[test]
fn backup_restore_pack_and_unpack_keep_the_files_sealed_and_openable_with_the_key() {
    use hayson_engine::database::{
        backup_database, pack_database, restore_database, unpack_database, BackupOptions, PackOptions, RestoreOptions,
        UnpackOptions,
    };
    let source = TempDir::new().unwrap();
    seed(source.path());
    let holder = parked(source.path());

    let backup = holder.path().join("backup");
    backup_database(
        source.path(),
        &backup,
        &BackupOptions {
            encryption: Some(keys(7)),
            ..BackupOptions::default()
        },
    )
    .unwrap();
    for (path, bytes) in files_under(&backup) {
        assert!(
            !contains(&bytes, "Kowalska") && !contains(&bytes, "secret-photo"),
            "{path}"
        );
    }
    assert!(backup_database(source.path(), &holder.path().join("b2"), &BackupOptions::default()).is_err());

    let restored = TempDir::new().unwrap();
    restore_database(
        &backup,
        restored.path(),
        &RestoreOptions {
            encryption: Some(keys(7)),
            ..RestoreOptions::default()
        },
    )
    .unwrap();
    let db = open_with(restored.path(), Some(keys(7))).unwrap();
    assert_eq!(db.table("people").unwrap().keys().count(), 2);
    db.close().unwrap();

    let pack = holder.path().join("p.gly");
    pack_database(
        source.path(),
        &pack,
        &PackOptions {
            encryption: Some(keys(7)),
            ..PackOptions::default()
        },
    )
    .unwrap();
    let target = TempDir::new().unwrap();
    assert!(
        unpack_database(&pack, target.path(), &UnpackOptions::default()).is_err(),
        "no key, no unpack"
    );
    unpack_database(
        &pack,
        target.path(),
        &UnpackOptions {
            encryption: Some(keys(7)),
            ..UnpackOptions::default()
        },
    )
    .unwrap();
    let db = open_with(target.path(), Some(keys(7))).unwrap();
    assert!(
        db.table("people")
            .unwrap()
            .verify_password("p1", "password", "correct horse battery")
            .unwrap()
            .ok
    );
}

#[test]
fn a_universe_and_projects_with_their_own_keys_travel_together() {
    use hayson_engine::database::{pack_database, unpack_database, PackInclude, PackOptions, UnpackOptions};
    let root = TempDir::new().unwrap();
    seed(root.path());
    let project = root.path().join("projects/a");
    fs::create_dir_all(&project).unwrap();
    let mut db = open_with(&project, Some(keys(9))).unwrap();
    db.create_table("notes", notes_schema()).unwrap();
    db.table_mut("notes")
        .unwrap()
        .put(&json!({ "id": "n1", "kind": "diary", "text": "project secret" }), false)
        .unwrap();
    db.close().unwrap();
    let holder = TempDir::new().unwrap();
    let pack = holder.path().join("all.gly");

    pack_database(
        root.path(),
        &pack,
        &PackOptions {
            encryption: Some(keys(7)),
            include: vec![PackInclude {
                path: "projects/a".into(),
                encryption: Some(keys(9)),
                exclude_tables: vec![],
            }],
            ..PackOptions::default()
        },
    )
    .unwrap();

    let target = TempDir::new().unwrap();
    let wrong_project_key = unpack_database(
        &pack,
        target.path(),
        &UnpackOptions {
            encryption: Some(keys(7)),
            include_encryption: vec![("projects/a".into(), keys(7))],
            ..UnpackOptions::default()
        },
    );
    assert!(wrong_project_key.is_err(), "the Project opened with the Universe's key");
    assert_eq!(
        fs::read_dir(target.path()).unwrap().count(),
        0,
        "nothing was left behind"
    );

    unpack_database(
        &pack,
        target.path(),
        &UnpackOptions {
            encryption: Some(keys(7)),
            include_encryption: vec![("projects/a".into(), keys(9))],
            ..UnpackOptions::default()
        },
    )
    .unwrap();
    assert!(open_with(target.path(), Some(keys(7))).is_ok());
    assert!(open_with(&target.path().join("projects/a"), Some(keys(9))).is_ok());
    assert!(open_with(&target.path().join("projects/a"), Some(keys(7))).is_err());
}

#[test]
fn moving_an_encrypted_database_between_arrangements_is_refused_for_now() {
    use hayson_engine::database::{migrate_layout, LayoutMigrationOptions};
    let dir = TempDir::new().unwrap();
    seed(dir.path());
    let refused = migrate_layout(dir.path(), LayoutMigrationOptions::default());
    assert!(refused.is_err());
}

fn seed_plain(dir: &Path) {
    let mut db = open_with(dir, None).unwrap();
    db.create_table("people", people_schema()).unwrap();
    db.create_table("notes", notes_schema()).unwrap();
    let people = db.table_mut("people").unwrap();
    people
        .put(
            &json!({
                "id": "p1", "email": "ann@example.test", "name": "Ann Kowalska",
                "photo": { "content": "c2VjcmV0LXBob3Rv", "virtualPath": "/ann.png" },
                "password": "correct horse battery",
            }),
            false,
        )
        .unwrap();
    people
        .put(
            &json!({ "id": "p2", "email": "bob@example.test", "name": "Bob Nowak" }),
            false,
        )
        .unwrap();
    db.table_mut("notes")
        .unwrap()
        .put(
            &json!({ "id": "n1", "kind": "diary", "text": "dear diary, my secret is x" }),
            false,
        )
        .unwrap();
    db.close().unwrap();
}

fn image_name_of(dir: &Path) -> String {
    let db = open_with(dir, None).or_else(|_| open_with(dir, Some(keys(7)))).unwrap();
    let name = db.table("people").unwrap().get("p1").unwrap().unwrap()["photo"]["physicalPath"]
        .as_str()
        .unwrap()
        .to_string();
    db.close().unwrap();
    name
}

#[test]
fn a_plain_database_is_encrypted_in_place_and_decrypted_back() {
    use hayson_engine::database::{convert_encryption, verify, ConvertOptions, VerifyOptions};
    let dir = TempDir::new().unwrap();
    seed_plain(dir.path());
    let plain_name = image_name_of(dir.path());

    let result = convert_encryption(
        dir.path(),
        &ConvertOptions {
            to: Some(keys(7)),
            ..ConvertOptions::default()
        },
    )
    .unwrap();
    assert_eq!(result.images, 1);
    assert_eq!(result.lines_renamed, 1, "the one record naming the image was rewritten");

    for (path, bytes) in files_under(dir.path()) {
        for secret in ["Kowalska", "ann@example", "secret-photo", "dear diary"] {
            assert!(!contains(&bytes, secret), "{path} leaks {secret:?}");
        }
    }
    let sealed_name = image_name_of(dir.path());
    assert_ne!(sealed_name, plain_name, "the name is keyed now");
    assert!(dir.path().join("people.images").join(&sealed_name).is_file());
    assert!(
        !dir.path().join("people.images").join(&plain_name).exists(),
        "the plain image is gone"
    );
    let db = open_with(dir.path(), Some(keys(7))).unwrap();
    let people = db.table("people").unwrap();
    assert!(
        people
            .verify_password("p1", "password", "correct horse battery")
            .unwrap()
            .ok
    );
    assert_eq!(people.read_image(&sealed_name).unwrap(), Some(b"secret-photo".to_vec()));
    assert_eq!(people.keys().count(), 2);
    db.close().unwrap();
    let report = verify(
        dir.path(),
        &VerifyOptions {
            encryption: Some(keys(7)),
            ..VerifyOptions::default()
        },
    );
    assert!(report.problems.is_empty(), "{:?}", report.problems);

    convert_encryption(
        dir.path(),
        &ConvertOptions {
            from: Some(keys(7)),
            ..ConvertOptions::default()
        },
    )
    .unwrap();
    assert_eq!(
        image_name_of(dir.path()),
        plain_name,
        "the plain name is the content hash again"
    );
    let db = open_with(dir.path(), None).unwrap();
    assert_eq!(
        db.table("people").unwrap().get("p2").unwrap().unwrap()["name"],
        json!("Bob Nowak")
    );
    assert!(fs::read_to_string(dir.path().join("people.jsonl"))
        .unwrap()
        .contains("Kowalska"));
    assert!(!dir.path().join(".restore-old").exists() && !dir.path().join(".restore-new").exists());
}

#[test]
fn rotating_the_key_re_seals_everything_and_the_old_key_stops_working() {
    use hayson_engine::database::{convert_encryption, verify, ConvertOptions, VerifyOptions};
    let dir = TempDir::new().unwrap();
    seed(dir.path());

    convert_encryption(
        dir.path(),
        &ConvertOptions {
            from: Some(keys(7)),
            to: Some(keys(8)),
            ..ConvertOptions::default()
        },
    )
    .unwrap();

    assert!(open_with(dir.path(), Some(keys(7))).is_err());
    let db = open_with(dir.path(), Some(keys(8))).unwrap();
    assert_eq!(db.table("people").unwrap().keys().count(), 2);
    assert_eq!(
        db.table("notes").unwrap().get("n1").unwrap().unwrap()["kind"],
        json!("diary")
    );
    let name = db.table("people").unwrap().list_images().unwrap()[0]
        .physical_path
        .clone();
    assert_eq!(
        db.table("people").unwrap().read_image(&name).unwrap(),
        Some(b"secret-photo".to_vec())
    );
    db.close().unwrap();
    let report = verify(
        dir.path(),
        &VerifyOptions {
            encryption: Some(keys(8)),
            ..VerifyOptions::default()
        },
    );
    assert!(report.problems.is_empty(), "{:?}", report.problems);
    assert_eq!(
        fs::read_dir(dir.path().join("people.images")).unwrap().count(),
        1,
        "the image under the old name is gone"
    );
}

#[test]
fn a_conversion_with_the_wrong_keys_or_nothing_to_do_changes_nothing() {
    use hayson_engine::database::{convert_encryption, ConvertOptions};
    let dir = TempDir::new().unwrap();
    seed(dir.path());
    let before = files_under(dir.path());

    let wrong = convert_encryption(
        dir.path(),
        &ConvertOptions {
            from: Some(keys(9)),
            to: Some(keys(8)),
            ..ConvertOptions::default()
        },
    );
    let plain_as_sealed = convert_encryption(
        dir.path(),
        &ConvertOptions {
            to: Some(keys(8)),
            ..ConvertOptions::default()
        },
    );
    let nothing = convert_encryption(dir.path(), &ConvertOptions::default());

    assert!(matches!(wrong, Err(DatabaseError::Cipher(_))), "{wrong:?}");
    assert!(plain_as_sealed.is_err() && nothing.is_err());
    let mut after = files_under(dir.path());
    let mut before = before;
    before.sort();
    after.sort();
    assert_eq!(before, after, "no file changed");
}