hayson-engine 0.27.0

//! Packing a database into one zip archive and unpacking it again, from outside the crate.

use std::fs;
use std::io::Write;
use std::path::Path;

use hayson_engine::database::{
    pack_database, read_pack_entry, unpack_database, Database, PackInclude, PackLimits, PackOptions, UnpackOptions,
    BACKUP_MANIFEST,
};
use serde_json::json;
use tempfile::TempDir;
use zip::write::SimpleFileOptions;
use zip::{CompressionMethod, ZipWriter};

fn seeded(root: &Path) {
    let mut database = Database::open(root).unwrap();
    database
        .create_table(
            "docs",
            json!({
                "schemaVersion": 1,
                "table": "docs",
                "primaryKey": "id",
                "fields": { "id": { "type": "string" }, "cover": { "type": "image" } },
            }),
        )
        .unwrap();
    database
        .table_mut("docs")
        .unwrap()
        .put(
            &json!({ "id": "d1", "cover": { "content": "Ynl0ZXM=", "virtualPath": "/a.png" } }),
            false,
        )
        .unwrap();
    database.close().unwrap();
}

fn count(root: &Path) -> usize {
    let database = Database::open(root).unwrap();
    let keys = database.table("docs").unwrap().keys().count();
    database.close().unwrap();
    keys
}

fn archive_with(path: &Path, entries: &[(&str, &[u8], CompressionMethod)]) {
    let mut writer = ZipWriter::new(fs::File::create(path).unwrap());
    for (name, bytes, method) in entries {
        writer
            .start_file(*name, SimpleFileOptions::default().compression_method(*method))
            .unwrap();
        writer.write_all(bytes).unwrap();
    }
    writer.finish().unwrap();
}

/// Declares the second entry of a stored archive as deflated (method 8) in both its headers; the
/// crate under test is built without a compressor, so this is the only way to make such an archive.
fn mark_second_entry_deflated(path: &Path) {
    let mut bytes = fs::read(path).unwrap();
    let nth = |bytes: &[u8], signature: &[u8; 4], n: usize| {
        (0..bytes.len() - 4)
            .filter(|i| &bytes[*i..*i + 4] == signature)
            .nth(n)
            .unwrap()
    };
    let local = nth(&bytes, b"PK\x03\x04", 1);
    bytes[local + 8] = 8;
    let central = nth(&bytes, b"PK\x01\x02", 1);
    bytes[central + 10] = 8;
    fs::write(path, bytes).unwrap();
}

fn manifest_for(files: &[(&str, u64)]) -> Vec<u8> {
    let listed: Vec<_> = files.iter().map(|(p, b)| json!({ "path": p, "bytes": b })).collect();
    serde_json::to_vec(&json!({ "format": 1, "metadataFile": "hayson.json", "files": listed })).unwrap()
}

#[test]
fn a_pack_unpacks_into_an_empty_directory_with_data_images_and_the_callers_entries() {
    let source = TempDir::new().unwrap();
    seeded(source.path());
    let holder = TempDir::new().unwrap();
    let pack = holder.path().join("project.gly");
    let options = PackOptions {
        extra: vec![("gly.json".to_string(), b"{\"v\":1}".to_vec())],
        ..PackOptions::default()
    };

    let made = pack_database(source.path(), &pack, &options).unwrap();
    assert_eq!(made.files, 3);
    assert!(!holder.path().join("project.gly.partial").exists());
    assert_eq!(
        read_pack_entry(&pack, "gly.json", &PackLimits::default()).unwrap(),
        Some(b"{\"v\":1}".to_vec())
    );
    assert_eq!(read_pack_entry(&pack, "nope", &PackLimits::default()).unwrap(), None);
    assert!(read_pack_entry(&pack, BACKUP_MANIFEST, &PackLimits::default())
        .unwrap()
        .is_some());

    let target = TempDir::new().unwrap();
    let restored = unpack_database(&pack, target.path(), &UnpackOptions::default()).unwrap();
    assert_eq!(restored.files, 3);
    assert_eq!(count(target.path()), 1);
    assert!(
        !target.path().join("gly.json").exists(),
        "extras are not database files"
    );
    let database = Database::open(target.path()).unwrap();
    let docs = database.table("docs").unwrap();
    let image = &docs.list_images().unwrap()[0];
    assert_eq!(docs.read_image(&image.physical_path).unwrap(), Some(b"bytes".to_vec()));
    database.close().unwrap();
}

#[test]
fn packing_refuses_an_existing_destination_and_a_clashing_extra() {
    let source = TempDir::new().unwrap();
    seeded(source.path());
    let holder = TempDir::new().unwrap();
    let pack = holder.path().join("p.gly");
    fs::write(&pack, b"x").unwrap();
    assert!(pack_database(source.path(), &pack, &PackOptions::default()).is_err());

    let fresh = holder.path().join("q.gly");
    for name in [BACKUP_MANIFEST, "hayson.json", "../x", "con"] {
        let options = PackOptions {
            extra: vec![(name.to_string(), b"x".to_vec())],
            ..PackOptions::default()
        };
        assert!(pack_database(source.path(), &fresh, &options).is_err(), "{name}");
        assert!(!fresh.exists());
    }
}

#[test]
fn unpacking_over_a_database_replaces_it_and_a_bad_pack_changes_nothing() {
    let root = TempDir::new().unwrap();
    seeded(root.path());
    let holder = TempDir::new().unwrap();
    let pack = holder.path().join("p.gly");
    pack_database(root.path(), &pack, &PackOptions::default()).unwrap();
    let mut database = Database::open(root.path()).unwrap();
    database
        .table_mut("docs")
        .unwrap()
        .put(&json!({ "id": "d2" }), false)
        .unwrap();
    database.close().unwrap();
    assert_eq!(count(root.path()), 2);

    let broken = holder.path().join("broken.gly");
    archive_with(
        &broken,
        &[
            (
                BACKUP_MANIFEST,
                &manifest_for(&[("hayson.json", 2), ("docs.jsonl", 8)]),
                CompressionMethod::Stored,
            ),
            ("hayson.json", b"{}", CompressionMethod::Stored),
            ("docs.jsonl", b"{broken\n", CompressionMethod::Stored),
        ],
    );
    assert!(unpack_database(&broken, root.path(), &UnpackOptions::default()).is_err());
    assert_eq!(count(root.path()), 2, "a refused pack leaves the database as it was");
    assert!(!root.path().join(".restore-new").exists());

    unpack_database(&pack, root.path(), &UnpackOptions::default()).unwrap();
    assert_eq!(count(root.path()), 1);
}

#[test]
fn hostile_archives_are_refused_before_anything_is_written() {
    let holder = TempDir::new().unwrap();
    let target = TempDir::new().unwrap();
    let ok_manifest = manifest_for(&[("hayson.json", 2)]);
    let stored = CompressionMethod::Stored;
    let cases: Vec<(&str, Vec<(&str, &[u8], CompressionMethod)>)> = vec![
        (
            "traversal",
            vec![
                (BACKUP_MANIFEST, &ok_manifest, stored),
                ("../hayson.json", b"{}", stored),
            ],
        ),
        (
            "absolute",
            vec![(BACKUP_MANIFEST, &ok_manifest, stored), ("/hayson.json", b"{}", stored)],
        ),
        (
            "backslash",
            vec![
                (BACKUP_MANIFEST, &ok_manifest, stored),
                ("a\\hayson.json", b"{}", stored),
            ],
        ),
        (
            "control",
            vec![(BACKUP_MANIFEST, &ok_manifest, stored), ("a\u{1}b", b"{}", stored)],
        ),
        (
            "device",
            vec![(BACKUP_MANIFEST, &ok_manifest, stored), ("aux.json", b"{}", stored)],
        ),
        (
            "duplicate-ignoring-case",
            vec![
                (BACKUP_MANIFEST, &ok_manifest, stored),
                ("hayson.json", b"{}", stored),
                ("HAYSON.json", b"{}", stored),
            ],
        ),
        (
            "compressed",
            vec![
                (BACKUP_MANIFEST, &ok_manifest, stored),
                ("hayson.json", b"{}{}{}{}{}{}{}{}", stored),
            ],
        ),
        ("no-manifest", vec![("hayson.json", b"{}", stored)]),
        (
            "size-mismatch",
            vec![(BACKUP_MANIFEST, &ok_manifest, stored), ("hayson.json", b"{} ", stored)],
        ),
        ("listed-but-absent", vec![(BACKUP_MANIFEST, &ok_manifest, stored)]),
    ];
    for (label, entries) in cases {
        let path = holder.path().join(format!("{label}.gly"));
        archive_with(&path, &entries);
        if label == "compressed" {
            mark_second_entry_deflated(&path);
        }
        let outcome = unpack_database(&path, target.path(), &UnpackOptions::default());
        assert!(outcome.is_err(), "{label} should be refused");
        assert_eq!(
            fs::read_dir(target.path()).unwrap().count(),
            0,
            "{label} wrote something"
        );
    }
    assert!(!target.path().parent().unwrap().join("hayson.json").exists());
}

#[test]
fn symlink_entries_and_limits_are_refused() {
    let holder = TempDir::new().unwrap();
    let target = TempDir::new().unwrap();
    let ok_manifest = manifest_for(&[("hayson.json", 2)]);

    let link = holder.path().join("link.gly");
    let mut writer = ZipWriter::new(fs::File::create(&link).unwrap());
    let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored);
    writer.start_file(BACKUP_MANIFEST, options).unwrap();
    writer.write_all(&ok_manifest).unwrap();
    writer.add_symlink("hayson.json", "/etc/passwd", options).unwrap();
    writer.finish().unwrap();
    assert!(unpack_database(&link, target.path(), &UnpackOptions::default()).is_err());

    let plain = holder.path().join("plain.gly");
    archive_with(
        &plain,
        &[
            (BACKUP_MANIFEST, &ok_manifest, CompressionMethod::Stored),
            ("hayson.json", b"{}", CompressionMethod::Stored),
        ],
    );
    for limits in [
        PackLimits {
            max_entries: 1,
            ..PackLimits::default()
        },
        PackLimits {
            max_entry_bytes: 1,
            ..PackLimits::default()
        },
        PackLimits {
            max_total_bytes: 1,
            ..PackLimits::default()
        },
        PackLimits {
            max_archive_bytes: 10,
            ..PackLimits::default()
        },
    ] {
        let outcome = unpack_database(
            &plain,
            target.path(),
            &UnpackOptions {
                limits,
                ..UnpackOptions::default()
            },
        );
        assert!(outcome.is_err());
    }
    assert!(unpack_database(
        &holder.path().join("missing.gly"),
        target.path(),
        &UnpackOptions::default()
    )
    .is_err());
    fs::write(holder.path().join("text.gly"), b"not a zip").unwrap();
    assert!(unpack_database(
        &holder.path().join("text.gly"),
        target.path(),
        &UnpackOptions::default()
    )
    .is_err());
}

#[test]
fn packing_enforces_the_limits_so_a_pack_is_always_readable() {
    let source = TempDir::new().unwrap();
    seeded(source.path());
    let holder = TempDir::new().unwrap();
    let pack = holder.path().join("p.gly");
    let options = PackOptions {
        limits: PackLimits {
            max_entry_bytes: 4,
            ..PackLimits::default()
        },
        ..PackOptions::default()
    };
    assert!(pack_database(source.path(), &pack, &options).is_err());
    assert!(!pack.exists());
    assert!(!holder.path().join("p.gly.partial").exists());
}

#[test]
fn excluded_tables_leave_their_data_out_but_stay_declared() {
    let source = TempDir::new().unwrap();
    seeded(source.path());
    let mut database = Database::open(source.path()).unwrap();
    database
        .create_table(
            "users",
            json!({
                "schemaVersion": 1,
                "table": "users",
                "primaryKey": "id",
                "fields": { "id": { "type": "string" }, "name": { "type": "string" } },
            }),
        )
        .unwrap();
    database
        .table_mut("users")
        .unwrap()
        .put(&json!({ "id": "u1", "name": "Ann" }), false)
        .unwrap();
    database.close().unwrap();
    let holder = TempDir::new().unwrap();
    let pack = holder.path().join("p.gly");

    let options = PackOptions {
        exclude_tables: vec!["users".to_string()],
        ..PackOptions::default()
    };
    let made = pack_database(source.path(), &pack, &options).unwrap();
    assert_eq!(made.files, 4, "catalog, docs log, one image, and an empty users log");
    let users_log = fs::metadata(source.path().join("users.jsonl")).unwrap().len();
    let full = pack_database(source.path(), &holder.path().join("full.gly"), &PackOptions::default()).unwrap();
    assert_eq!(made.bytes + users_log, full.bytes, "no users data was packed");

    let target = TempDir::new().unwrap();
    unpack_database(&pack, target.path(), &UnpackOptions::default()).unwrap();
    assert_eq!(fs::metadata(target.path().join("users.jsonl")).unwrap().len(), 0);
    let database = Database::open(target.path()).unwrap();
    assert_eq!(
        database.table("users").unwrap().keys().count(),
        0,
        "declared, but empty"
    );
    assert_eq!(database.table("docs").unwrap().keys().count(), 1);
    database.close().unwrap();

    let typo = PackOptions {
        exclude_tables: vec!["usres".to_string()],
        ..PackOptions::default()
    };
    assert!(pack_database(source.path(), &holder.path().join("q.gly"), &typo).is_err());
}

fn patched(path: &Path, edit: impl FnOnce(&mut Vec<u8>)) {
    let mut bytes = fs::read(path).unwrap();
    edit(&mut bytes);
    fs::write(path, bytes).unwrap();
}

fn nth_signature(bytes: &[u8], signature: &[u8; 4], n: usize) -> usize {
    (0..bytes.len() - 4)
        .filter(|i| &bytes[*i..*i + 4] == signature)
        .nth(n)
        .unwrap()
}

#[test]
fn archives_with_inconsistent_headers_or_stray_bytes_are_refused() {
    let source = TempDir::new().unwrap();
    seeded(source.path());
    let holder = TempDir::new().unwrap();
    let target = TempDir::new().unwrap();
    let good = holder.path().join("good.gly");
    pack_database(source.path(), &good, &PackOptions::default()).unwrap();
    // Sanity: the unmodified pack is fine, so what follows fails for the edit alone.
    let probe = TempDir::new().unwrap();
    unpack_database(&good, probe.path(), &UnpackOptions::default()).unwrap();

    let variant = |name: &str, edit: &dyn Fn(&mut Vec<u8>)| {
        let path = holder.path().join(name);
        fs::copy(&good, &path).unwrap();
        patched(&path, |bytes| edit(bytes));
        path
    };
    // Entries come in path order: docs.jsonl, hayson.json, then the image, then the manifest.
    let mismatch = variant("mismatch.gly", &|bytes| {
        let local = nth_signature(bytes, b"PK\x03\x04", 1);
        bytes[local + 30] = b'X';
    });
    let trailing = variant("trailing.gly", &|bytes| bytes.extend_from_slice(b"trailing junk"));
    let not_utf8 = variant("not-utf8.gly", &|bytes| {
        let local = nth_signature(bytes, b"PK\x03\x04", 1);
        bytes[local + 30] = 0xFF;
        let central = nth_signature(bytes, b"PK\x01\x02", 1);
        bytes[central + 46] = 0xFF;
    });
    // Two extra entries whose names differ in one byte, then made equal.
    let twins = holder.path().join("twins.gly");
    pack_database(
        source.path(),
        &twins,
        &PackOptions {
            extra: vec![
                ("aa.txt".to_string(), b"1".to_vec()),
                ("ab.txt".to_string(), b"2".to_vec()),
            ],
            ..PackOptions::default()
        },
    )
    .unwrap();
    patched(&twins, |bytes| {
        for signature in [b"PK\x03\x04", b"PK\x01\x02"] {
            let at = (0..bytes.len() - 10)
                .filter(|i| &bytes[*i..*i + 4] == signature)
                .find(|i| {
                    let offset = if signature == b"PK\x03\x04" { 30 } else { 46 };
                    &bytes[*i + offset..*i + offset + 6] == b"ab.txt"
                })
                .unwrap();
            let offset = if signature == b"PK\x03\x04" { 30 } else { 46 };
            bytes[at + offset + 1] = b'a';
        }
    });
    let duplicate = twins;

    for path in [&mismatch, &trailing, &not_utf8, &duplicate] {
        let outcome = unpack_database(path, target.path(), &UnpackOptions::default());
        let error = outcome.expect_err(&format!("{} should be refused", path.display()));
        assert!(
            !error.to_string().contains("does not verify"),
            "{}: refused only after extraction ({error})",
            path.display()
        );
        assert_eq!(fs::read_dir(target.path()).unwrap().count(), 0);
    }
}

#[test]
fn an_oversize_image_refuses_the_unpack_and_leaves_no_directory_behind() {
    let source = TempDir::new().unwrap();
    seeded(source.path());
    let holder = TempDir::new().unwrap();
    let pack = holder.path().join("p.gly");
    pack_database(source.path(), &pack, &PackOptions::default()).unwrap();
    let target = holder.path().join("fresh");

    let refused = unpack_database(
        &pack,
        &target,
        &UnpackOptions {
            max_image_bytes: Some(4), // the seeded image is 5 bytes
            ..UnpackOptions::default()
        },
    );

    assert!(refused.is_err());
    assert!(!target.exists(), "the directory the call created is gone again");
    let ok = unpack_database(
        &pack,
        &target,
        &UnpackOptions {
            max_image_bytes: Some(5),
            ..UnpackOptions::default()
        },
    );
    assert!(ok.is_ok());
}

#[test]
fn verify_does_not_flag_the_logs_of_a_nested_database_as_undeclared() {
    let root = TempDir::new().unwrap();
    seeded(root.path());
    let child = root.path().join("project");
    fs::create_dir_all(&child).unwrap();
    seeded(&child);

    let report = hayson_engine::database::verify(root.path(), &hayson_engine::database::VerifyOptions::default());

    assert!(
        !report.problems.iter().any(|p| p.code == "log-undeclared"),
        "{:?}",
        report.problems
    );
}

fn put_user(root: &Path, id: &str) {
    let mut database = Database::open(root).unwrap();
    if database.table("users").is_none() {
        database
            .create_table(
                "users",
                json!({
                    "schemaVersion": 1,
                    "table": "users",
                    "primaryKey": "id",
                    "fields": { "id": { "type": "string" } },
                }),
            )
            .unwrap();
    }
    database
        .table_mut("users")
        .unwrap()
        .put(&json!({ "id": id }), false)
        .unwrap();
    database.close().unwrap();
}

fn users(root: &Path) -> usize {
    let database = Database::open(root).unwrap();
    let count = database.table("users").map(|t| t.keys().count()).unwrap_or(0);
    database.close().unwrap();
    count
}

/// A Universe with two Projects: `users` in the Universe, `docs` (with an image) in each Project.
fn universe() -> TempDir {
    let root = TempDir::new().unwrap();
    seeded(root.path());
    put_user(root.path(), "u1");
    for project in ["projects/a", "projects/b"] {
        let dir = root.path().join(project);
        fs::create_dir_all(&dir).unwrap();
        seeded(&dir);
    }
    root
}

fn plain_includes() -> Vec<PackInclude> {
    ["projects/a", "projects/b"]
        .iter()
        .map(|path| PackInclude {
            path: path.to_string(),
            encryption: None,
            exclude_tables: vec![],
        })
        .collect()
}

fn combined() -> PackOptions {
    PackOptions {
        include: vec![
            PackInclude {
                path: "projects/a".into(),
                encryption: None,
                exclude_tables: vec![],
            },
            PackInclude {
                path: "projects/b".into(),
                encryption: None,
                exclude_tables: vec!["docs".into()],
            },
        ],
        exclude_tables: vec!["users".into()],
        ..PackOptions::default()
    }
}

#[test]
fn a_universe_and_its_projects_travel_in_one_pack_with_a_manifest_each() {
    let source = universe();
    let holder = TempDir::new().unwrap();
    let pack = holder.path().join("all.gly");

    let made = pack_database(source.path(), &pack, &combined()).unwrap();

    let paths: Vec<&str> = made.databases.iter().map(|d| d.path.as_str()).collect();
    assert_eq!(paths, vec!["", "projects/a", "projects/b"]);
    for name in [
        BACKUP_MANIFEST,
        "projects/a/hayson-backup.json",
        "projects/b/hayson-backup.json",
    ] {
        assert!(
            read_pack_entry(&pack, name, &PackLimits::default()).unwrap().is_some(),
            "{name}"
        );
    }

    let target = TempDir::new().unwrap();
    unpack_database(&pack, target.path(), &UnpackOptions::default()).unwrap();
    assert_eq!(users(target.path()), 0, "the Universe's users were excluded");
    let a = Database::open(&target.path().join("projects/a")).unwrap();
    assert_eq!(a.table("docs").unwrap().keys().count(), 1);
    assert_eq!(a.table("docs").unwrap().list_images().unwrap().len(), 1);
    a.close().unwrap();
    let b = Database::open(&target.path().join("projects/b")).unwrap();
    assert_eq!(
        b.table("docs").unwrap().keys().count(),
        0,
        "project b's docs were excluded"
    );
    b.close().unwrap();
}

#[test]
fn a_bad_nested_database_refuses_the_whole_unpack_and_changes_nothing() {
    let source = universe();
    let holder = TempDir::new().unwrap();
    let pack = holder.path().join("all.gly");
    pack_database(
        source.path(),
        &pack,
        &PackOptions {
            include: plain_includes(),
            ..PackOptions::default()
        },
    )
    .unwrap();
    // Corrupt project b's docs log inside the archive by re-packing from a damaged copy.
    fs::write(source.path().join("projects/b/docs.jsonl"), b"{broken\n").unwrap();
    let damaged = holder.path().join("damaged.gly");
    pack_database(
        source.path(),
        &damaged,
        &PackOptions {
            include: plain_includes(),
            ..PackOptions::default()
        },
    )
    .unwrap();

    let target = TempDir::new().unwrap();
    unpack_database(&pack, target.path(), &UnpackOptions::default()).unwrap();
    put_user(target.path(), "later");
    assert_eq!(users(target.path()), 2);

    let refused = unpack_database(&damaged, target.path(), &UnpackOptions::default());

    assert!(refused.is_err());
    assert_eq!(users(target.path()), 2, "the Universe was not swapped either");
    assert!(!target.path().join(".restore-new").exists());
    assert!(!target.path().join(".restore-old").exists());
}

#[test]
fn unpacking_over_existing_databases_replaces_all_of_them() {
    let source = universe();
    let holder = TempDir::new().unwrap();
    let pack = holder.path().join("all.gly");
    pack_database(
        source.path(),
        &pack,
        &PackOptions {
            include: combined().include,
            ..PackOptions::default()
        },
    )
    .unwrap();
    put_user(source.path(), "u2");
    assert_eq!(users(source.path()), 2);

    unpack_database(&pack, source.path(), &UnpackOptions::default()).unwrap();

    assert_eq!(users(source.path()), 1);
}

#[test]
fn includes_that_repeat_overlap_or_are_not_databases_are_refused() {
    let source = universe();
    let holder = TempDir::new().unwrap();
    for paths in [
        vec!["projects/a", "projects/a"],
        vec!["projects", "projects/a"],
        vec!["nope"],
        vec!["../x"],
    ] {
        let options = PackOptions {
            include: paths
                .iter()
                .map(|path| PackInclude {
                    path: path.to_string(),
                    encryption: None,
                    exclude_tables: vec![],
                })
                .collect(),
            ..PackOptions::default()
        };
        let dest = holder.path().join("x.gly");
        assert!(pack_database(source.path(), &dest, &options).is_err(), "{paths:?}");
        assert!(!dest.exists());
    }
}

#[test]
fn an_entry_whose_bytes_changed_fails_its_checksum_as_a_refusal_of_the_archive() {
    let source = TempDir::new().unwrap();
    seeded(source.path());
    let holder = TempDir::new().unwrap();
    let pack = holder.path().join("p.gly");
    pack_database(source.path(), &pack, &PackOptions::default()).unwrap();
    // Flip a byte inside the stored data of the first entry, leaving every header intact.
    patched(&pack, |bytes| {
        let local = nth_signature(bytes, b"PK\x03\x04", 0);
        let name_len = u16::from_le_bytes([bytes[local + 26], bytes[local + 27]]) as usize;
        let extra_len = u16::from_le_bytes([bytes[local + 28], bytes[local + 29]]) as usize;
        let data = local + 30 + name_len + extra_len;
        bytes[data] ^= 0xff;
    });
    let target = TempDir::new().unwrap();

    let error = unpack_database(&pack, target.path(), &UnpackOptions::default()).unwrap_err();

    assert!(
        matches!(error, hayson_engine::database::DatabaseError::Schema(_)),
        "{error:?}"
    );
    assert!(error.to_string().contains("damaged"), "{error}");
}