use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use base64::engine::general_purpose::STANDARD;
use base64::Engine as _;
use hayson_engine::crypto::{CipherOptions, FieldCipher};
use hayson_engine::database::{Database, OpenOptions};
use hayson_engine::query::{ComparisonOp, QueryBuilder, QuerySpec, SearchMode, SortDirection};
use hayson_engine::schema::parse_schema;
use hayson_engine::table::Table;
use serde_json::{json, Map, Value};
fn fixtures_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("..")
.join("fixtures")
.join("format2")
}
fn read_json(path: &Path) -> Value {
serde_json::from_slice(&fs::read(path).unwrap_or_else(|error| panic!("{}: {error}", path.display()))).unwrap()
}
fn expectations() -> Value {
read_json(&fixtures_root().join("expectations.json"))
}
fn copy_tree(from: &Path, to: &Path) {
fs::create_dir_all(to).unwrap();
for entry in fs::read_dir(from).unwrap() {
let entry = entry.unwrap();
let target = to.join(entry.file_name());
if entry.file_type().unwrap().is_dir() {
copy_tree(&entry.path(), &target);
} else {
fs::copy(entry.path(), target).unwrap();
}
}
}
fn copied(name: &str, into: &Path) -> PathBuf {
let target = into.join(name);
copy_tree(&fixtures_root().join(name), &target);
target
}
fn keyring() -> HashMap<String, Vec<u8>> {
let document = read_json(&fixtures_root().join("crypto-vectors").join("keys.json"));
document["keys"]
.as_object()
.unwrap()
.iter()
.map(|(id, material)| (id.clone(), STANDARD.decode(material.as_str().unwrap()).unwrap()))
.collect()
}
fn cipher_options(expectation: &Value) -> Option<CipherOptions> {
let cypher = expectation.get("cypher")?;
Some(CipherOptions {
keys: keyring(),
key_id: cypher["keyId"].as_str().map(str::to_string),
on_missing_key: None,
})
}
fn open_readonly(dir: &Path, expectation: &Value) -> Result<Database, String> {
Database::open_with(
dir,
OpenOptions {
readonly: true,
cypher: cipher_options(expectation),
..OpenOptions::default()
},
)
.map_err(|error| error.to_string())
}
fn op_of(name: &str) -> ComparisonOp {
match name {
"eq" => ComparisonOp::Eq,
"ne" => ComparisonOp::Ne,
"gt" => ComparisonOp::Gt,
"gte" => ComparisonOp::Gte,
"lt" => ComparisonOp::Lt,
"lte" => ComparisonOp::Lte,
"in" => ComparisonOp::In,
"contains" => ComparisonOp::Contains,
"startsWith" => ComparisonOp::StartsWith,
other => panic!("expectations.json names an operator this reader does not know: {other}"),
}
}
fn spec_of(query: &Value) -> QuerySpec {
let mut builder = QueryBuilder::new();
for leaf in query["where"].as_array().unwrap_or(&Vec::new()) {
builder = builder.r#where(
leaf["field"].as_str().unwrap(),
op_of(leaf["op"].as_str().unwrap()),
leaf["value"].clone(),
);
}
if let Some(text) = query.get("search").and_then(Value::as_str) {
builder = builder.search(text, SearchMode::All);
}
if let Some(field) = query.get("orderBy").and_then(Value::as_str) {
builder = builder.order_by(field, SortDirection::Asc);
}
builder.spec()
}
fn record_json(record: &std::collections::BTreeMap<String, Value>) -> Value {
Value::Object(record.clone().into_iter().collect())
}
fn envelope_key_id(value: &Value) -> Option<&str> {
let text = value.as_str()?;
let parts: Vec<&str> = text.strip_prefix('$')?.split('$').collect();
if parts.len() != 5 || parts[0] != "cypher" || parts[1] != "v1" {
return None;
}
Some(parts[2])
}
fn sorted_strings(values: &Value) -> Vec<String> {
let mut out: Vec<String> = values
.as_array()
.unwrap()
.iter()
.map(|value| value.as_str().unwrap().to_string())
.collect();
out.sort();
out
}
fn case_names() -> Vec<String> {
expectations()["expectations"]
.as_object()
.unwrap()
.keys()
.cloned()
.collect()
}
#[test]
fn every_case_carries_a_sentence_saying_what_it_proves() {
let recorded = expectations();
let cases = recorded["expectations"].as_object().unwrap();
assert!(!cases.is_empty(), "expectations.json describes no case at all");
for (name, expectation) in cases {
let proves = expectation["proves"].as_str().unwrap_or_default();
assert!(proves.len() > 20, "case {name} says nothing about what it proves");
assert!(
fixtures_root().join(name).is_dir(),
"case {name} is described but its directory is missing"
);
}
}
#[test]
fn every_fixture_file_is_present_and_the_length_it_was_recorded_at() {
let recorded = expectations();
for (key, sum) in recorded["checksums"].as_object().unwrap() {
let path = fixtures_root().join(key);
let bytes = fs::read(&path).unwrap_or_else(|error| panic!("{}: {error}", path.display()));
assert_eq!(
bytes.len() as u64,
sum["bytes"].as_u64().unwrap(),
"{key} changed size since it was recorded"
);
}
}
#[test]
fn every_case_opens_exactly_as_recorded_and_reads_back_what_it_recorded() {
let recorded = expectations();
let temp = tempfile::tempdir().unwrap();
for name in case_names() {
let expectation = &recorded["expectations"][&name];
let dir = copied(&name, temp.path());
let open = &expectation["open"];
if open["ok"] == json!(false) {
let wanted = open["errorContains"]["engine"].as_str().unwrap();
let error = match open_readonly(&dir, expectation) {
Ok(_) => panic!("{name}: opened, though expectations.json says it must be refused"),
Err(error) => error,
};
assert!(
error.contains(wanted),
"{name}: refusal does not mention \"{wanted}\": {error}"
);
continue;
}
let db = open_readonly(&dir, expectation)
.unwrap_or_else(|error| panic!("{name}: expected to open, was refused: {error}"));
for (table_name, table_expectation) in expectation["tables"].as_object().unwrap_or(&Map::new()) {
let table = db
.table(table_name)
.unwrap_or_else(|| panic!("{name}: the catalog does not declare {table_name}"));
for expected in table_expectation["records"].as_array().unwrap() {
let key = expected["id"].as_str().unwrap();
let stored = table
.get(key)
.unwrap_or_else(|error| panic!("{name}: reading {table_name}/{key}: {error}"))
.unwrap_or_else(|| panic!("{name}: {table_name}/{key} is missing"));
assert_eq!(
record_json(&stored),
*expected,
"{name}: {table_name}/{key} does not read back as recorded"
);
}
for key in table_expectation["absentKeys"].as_array().unwrap_or(&Vec::new()) {
let key = key.as_str().unwrap();
assert!(
table.get(key).unwrap().is_none(),
"{name}: {table_name}/{key} is readable, though it was deleted"
);
}
}
for query in expectation["queries"].as_array().unwrap_or(&Vec::new()) {
let table = db.table(query["table"].as_str().unwrap()).unwrap();
let mut keys = table.query(&spec_of(query)).unwrap().keys;
keys.sort();
assert_eq!(
keys,
sorted_strings(&query["expectKeys"]),
"{name}: query \"{}\" answered differently",
query["name"].as_str().unwrap_or_default()
);
}
}
}
#[test]
fn reading_a_fixture_leaves_every_log_byte_where_it_was() {
let recorded = expectations();
let temp = tempfile::tempdir().unwrap();
for name in case_names() {
let expectation = &recorded["expectations"][&name];
if expectation["open"]["ok"] == json!(false) {
continue;
}
let dir = copied(&name, temp.path());
let logs: Vec<(String, Vec<u8>)> = recorded["checksums"]
.as_object()
.unwrap()
.keys()
.filter(|key| key.starts_with(&format!("{name}/")) && key.ends_with(".jsonl"))
.map(|key| {
let relative = key.strip_prefix(&format!("{name}/")).unwrap().to_string();
let bytes = fs::read(dir.join(&relative)).unwrap();
(relative, bytes)
})
.collect();
{
let db = open_readonly(&dir, expectation).unwrap();
for table_name in db.table_names() {
let _ = db.table(table_name).unwrap().keys().count();
}
}
for (relative, before) in logs {
assert_eq!(
fs::read(dir.join(&relative)).unwrap(),
before,
"{name}: a readonly open rewrote {relative}"
);
}
}
}
#[test]
fn the_partitioned_case_keeps_one_sequence_across_its_files_and_routes_the_tombstone() {
let recorded = expectations();
let expectation = &recorded["expectations"]["partitioned-log"];
let partitions = &expectation["partitions"];
let dir = fixtures_root().join("partitioned-log");
let mut seen: Vec<u64> = Vec::new();
for (file, sequences) in partitions["seqByPartition"].as_object().unwrap() {
let path = dir.join("notes").join(file);
let on_disk: Vec<u64> = fs::read_to_string(&path)
.unwrap()
.lines()
.map(|line| serde_json::from_str::<Value>(line).unwrap()["_seq"].as_u64().unwrap())
.collect();
assert_eq!(
on_disk,
sequences
.as_array()
.unwrap()
.iter()
.map(|value| value.as_u64().unwrap())
.collect::<Vec<_>>(),
"partitioned-log: {file} does not carry the recorded sequence numbers"
);
seen.extend(on_disk);
}
seen.sort_unstable();
let expected: Vec<u64> = (1..=seen.len() as u64).collect();
assert_eq!(seen, expected, "_seq is not one counter across the partition files");
let tombstone_file = partitions["tombstoneIn"].as_str().unwrap();
let tombstones: Vec<Value> = fs::read_to_string(fixtures_root().join("partitioned-log").join(tombstone_file))
.unwrap()
.lines()
.map(|line| serde_json::from_str::<Value>(line).unwrap())
.filter(|entry| entry["_op"] == json!("del"))
.collect();
assert_eq!(
tombstones.len(),
1,
"the tombstone must sit in the partition its record lived in, and only there"
);
let header: Value = serde_json::from_str(
fs::read_to_string(dir.join("notes.idx"))
.unwrap()
.lines()
.next()
.unwrap(),
)
.unwrap();
let parts: Vec<String> = header["parts"]
.as_array()
.unwrap()
.iter()
.map(|part| part["p"].as_str().unwrap().to_string())
.collect();
assert_eq!(parts, sorted_strings(&partitions["snapshotParts"]));
}
#[test]
fn the_snapshot_cases_land_on_the_outcome_the_format_doc_prescribes() {
let recorded = expectations();
let temp = tempfile::tempdir().unwrap();
for name in case_names() {
let expectation = &recorded["expectations"][&name];
let Some(snapshot) = expectation.get("snapshot") else {
continue;
};
let dir = copied(&name, temp.path());
let file = snapshot["file"].as_str().unwrap();
let header: Value =
serde_json::from_str(fs::read_to_string(dir.join(file)).unwrap().lines().next().unwrap()).unwrap();
if let Some(marker) = snapshot.get("headerMarker").and_then(Value::as_str) {
assert!(
header.get(marker).is_some(),
"{name}: {file} has no header key \"{marker}\""
);
}
if snapshot.get("hasPrefixHash") == Some(&json!(false)) {
assert!(
header.get("prefixHash").is_none(),
"{name}: {file} still carries a prefixHash"
);
}
match snapshot.get("outcome").and_then(Value::as_str) {
Some("mismatch") => {
let error = match open_readonly(&dir, expectation) {
Ok(_) => panic!("{name}: a mismatched snapshot must be a refusal"),
Err(error) => error,
};
assert!(error.contains("content fingerprint"), "{name}: {error}");
assert!(
dir.join(file).exists(),
"{name}: the mismatched snapshot was deleted instead of kept"
);
}
Some("stale") => {
let db = open_readonly(&dir, expectation)
.unwrap_or_else(|error| panic!("{name}: a stale snapshot must be silent, got: {error}"));
let table = db.table_names()[0].to_string();
assert!(
db.table(&table).unwrap().keys().count() > 0,
"{name}: replay found nothing"
);
}
_ => {
let header_watermark = header["watermark"].as_u64().unwrap();
if snapshot.get("watermarkEqualsLogSize") == Some(&json!(true)) {
let log = dir.join(format!("{}.jsonl", header["table"].as_str().unwrap()));
assert_eq!(
fs::metadata(&log).unwrap().len(),
header_watermark,
"{name}: the snapshot is not level with its log"
);
}
}
}
}
}
#[test]
fn the_catalog_is_written_in_the_canonical_order() {
let recorded = expectations();
let expectation = &recorded["expectations"]["catalog-determinism"];
let catalog_expectation = &expectation["catalog"];
let text = fs::read_to_string(fixtures_root().join("catalog-determinism").join("hayson.json")).unwrap();
let catalog: Value = serde_json::from_str(&text).unwrap();
let top_level: Vec<&str> = catalog.as_object().unwrap().keys().map(String::as_str).collect();
let expected_top: Vec<String> = catalog_expectation["topLevelKeys"]
.as_array()
.unwrap()
.iter()
.map(|value| value.as_str().unwrap().to_string())
.collect();
assert_eq!(top_level, expected_top, "the catalog's top-level keys moved");
let tables: Vec<&str> = catalog["tables"]
.as_object()
.unwrap()
.keys()
.map(String::as_str)
.collect();
assert_eq!(
tables,
catalog_expectation["tableOrder"]
.as_array()
.unwrap()
.iter()
.map(|value| value.as_str().unwrap())
.collect::<Vec<_>>(),
"the tables are not sorted"
);
for (table, fields) in catalog_expectation["fieldOrder"].as_object().unwrap() {
let on_disk: Vec<&str> = catalog["tables"][table]["fields"]
.as_object()
.unwrap()
.keys()
.map(String::as_str)
.collect();
assert_eq!(
on_disk,
fields
.as_array()
.unwrap()
.iter()
.map(|v| v.as_str().unwrap())
.collect::<Vec<_>>(),
"the fields of {table} are not sorted"
);
}
assert_eq!(
text.ends_with('\n'),
catalog_expectation["endsWithNewline"].as_bool().unwrap(),
"the trailing newline of hayson.json changed"
);
}
#[test]
fn the_cypher_log_carries_the_typescript_envelopes_under_two_key_ids() {
let recorded = expectations();
let expectation = &recorded["expectations"]["cypher-rotation"];
let cypher_log = &expectation["cypherLog"];
let lines = fs::read_to_string(
fixtures_root()
.join("cypher-rotation")
.join(cypher_log["file"].as_str().unwrap()),
)
.unwrap();
let mut key_ids: HashMap<String, Option<String>> = HashMap::new();
for line in lines.lines() {
let entry: Value = serde_json::from_str(line).unwrap();
let key = entry["_k"].as_str().unwrap().to_string();
let token = &entry["token"];
key_ids.insert(key.clone(), envelope_key_id(token).map(str::to_string));
if let Some(expected) = cypher_log["envelopes"].get(&key) {
for field in cypher_log["fields"].as_array().unwrap() {
let field = field.as_str().unwrap();
assert_eq!(
entry[field], expected[field],
"cypher-rotation: {key}.{field} is no longer the envelope the TS engine wrote"
);
}
} else {
assert!(
token.is_null(),
"cypher-rotation: {key} was expected to hold a null token"
);
}
}
for (key, expected) in cypher_log["keyIdByRecord"].as_object().unwrap() {
assert_eq!(
key_ids.get(key).cloned().flatten().as_deref(),
expected.as_str(),
"cypher-rotation: {key} is sealed under the wrong key id"
);
}
}
#[test]
fn every_golden_vector_decrypts_to_the_value_the_typescript_engine_sealed() {
let vectors = read_json(&fixtures_root().join("crypto-vectors").join("vectors.json"));
let keys = keyring();
for vector in vectors["vectors"].as_array().unwrap() {
let name = vector["name"].as_str().unwrap();
let cipher = FieldCipher::new(CipherOptions {
keys: keys.clone(),
key_id: Some(vector["keyId"].as_str().unwrap().to_string()),
on_missing_key: None,
})
.unwrap();
let envelope = vector["envelope"].as_str().unwrap();
assert_eq!(
envelope_key_id(&json!(envelope)),
vector["keyId"].as_str(),
"{name}: the envelope names another key id than the vector"
);
let plain = cipher
.decrypt(
vector["table"].as_str().unwrap(),
vector["key"].as_str().unwrap(),
vector["field"].as_str().unwrap(),
envelope,
)
.unwrap_or_else(|error| panic!("{name}: this build cannot open a TS envelope: {error}"))
.unwrap();
assert_eq!(plain, vector["value"], "{name}: decrypts to another value");
let body = envelope.split('$').next_back().unwrap();
assert_eq!(
base64::engine::general_purpose::URL_SAFE_NO_PAD
.decode(body)
.unwrap()
.len() as u64,
vector["bodyBytes"].as_u64().unwrap(),
"{name}: the ciphertext is no longer the recorded length"
);
}
}
#[test]
fn moving_a_golden_envelope_to_another_table_record_or_field_fails_authentication() {
let vectors = read_json(&fixtures_root().join("crypto-vectors").join("vectors.json"));
let keys = keyring();
let vector = vectors["vectors"]
.as_array()
.unwrap()
.iter()
.find(|vector| vector["name"] == json!("string-ascii"))
.unwrap();
let cipher = FieldCipher::new(CipherOptions {
keys: keys.clone(),
key_id: Some("k-2026-01".to_string()),
on_missing_key: None,
})
.unwrap();
let envelope = vector["envelope"].as_str().unwrap();
for (table, key, field) in [
("other", "s1", "token"),
("secrets", "s2", "token"),
("secrets", "s1", "pin"),
] {
let error = cipher.decrypt(table, key, field, envelope).expect_err(&format!(
"{table}/{key}/{field} opened an envelope that was not sealed for it"
));
assert!(error.reason.contains("authentication tag"), "{}", error.reason);
}
}
#[test]
fn a_key_that_is_not_on_hand_is_refused_by_name_rather_than_guessed() {
let vectors = read_json(&fixtures_root().join("crypto-vectors").join("vectors.json"));
let vector = vectors["vectors"]
.as_array()
.unwrap()
.iter()
.find(|vector| vector["name"] == json!("rotated-to-the-second-key"))
.unwrap();
let keys = keyring();
let only_the_retired_key = FieldCipher::new(CipherOptions {
keys: HashMap::from([("k-2026-01".to_string(), keys["k-2026-01"].clone())]),
key_id: None,
on_missing_key: None,
})
.unwrap();
let error = only_the_retired_key
.decrypt("secrets", "s1", "token", vector["envelope"].as_str().unwrap())
.unwrap_err();
assert_eq!(error.key_id.as_deref(), Some("k-2026-07"));
assert!(error.reason.contains("was not given"), "{}", error.reason);
}
fn seals_today_what_this_pair_recorded_in_the_clear(vector: &Value, keys: &HashMap<String, Vec<u8>>) {
let table = vector["table"].as_str().unwrap();
let key = vector["key"].as_str().unwrap();
let field = vector["field"].as_str().unwrap();
let key_id = vector["keyId"].as_str().unwrap();
let mut fields = Map::new();
fields.insert("id".to_string(), json!({ "type": "string" }));
fields.insert(field.to_string(), json!({ "type": "string", "cypher": true }));
let schema = parse_schema(&json!({
"schemaVersion": 1,
"table": table,
"primaryKey": "id",
"fields": Value::Object(fields),
}))
.unwrap();
let cipher = FieldCipher::new(CipherOptions {
keys: keys.clone(),
key_id: Some(key_id.to_string()),
on_missing_key: None,
})
.unwrap();
let temp = tempfile::tempdir().unwrap();
let log = temp.path().join(format!("{table}.jsonl"));
let mut document = Map::new();
document.insert("id".to_string(), json!(key));
document.insert(field.to_string(), vector["value"].clone());
{
let mut handle = Table::open(schema, &log).unwrap();
handle.set_cipher(Some(cipher.clone()));
handle.put(&Value::Object(document), false).unwrap();
}
let text = fs::read_to_string(&log).unwrap();
let line: Value = serde_json::from_str(text.lines().next_back().unwrap()).unwrap();
assert_ne!(
line[field], vector["value"],
"the engine stored a plaintext shaped like an envelope in the clear again"
);
assert_eq!(
cipher
.decrypt(table, key, field, line[field].as_str().unwrap())
.unwrap()
.unwrap(),
vector["value"],
"what it stored instead does not open back into the value that was written"
);
}
#[test]
fn the_recorded_cross_check_says_both_engines_wrote_the_same_bytes() {
let cross_check = read_json(&fixtures_root().join("crypto-vectors").join("cross-check.json"));
assert_eq!(
cross_check["identical"],
json!(true),
"the recorded TS/Rust comparison is not clean — see fixtures/format2/README.md"
);
let keys = keyring();
let vectors = read_json(&fixtures_root().join("crypto-vectors").join("vectors.json"));
let pairs = cross_check["pairs"].as_array().unwrap();
assert_eq!(pairs.len(), vectors["vectors"].as_array().unwrap().len());
for pair in pairs {
let name = pair["name"].as_str().unwrap();
let vector = vectors["vectors"]
.as_array()
.unwrap()
.iter()
.find(|vector| vector["name"] == json!(name))
.unwrap_or_else(|| panic!("cross-check.json names {name}, vectors.json does not"));
assert_eq!(pair["identical"], json!(true), "{name}: the two engines disagreed");
if pair.get("sealSkipped") == Some(&json!(true)) {
assert_eq!(pair["rustStored"], pair["tsStored"], "{name}");
assert_eq!(
pair["rustStored"], vector["value"],
"{name}: the pair no longer records the plaintext it was captured about"
);
seals_today_what_this_pair_recorded_in_the_clear(vector, &keys);
continue;
}
let cipher = FieldCipher::new(CipherOptions {
keys: keys.clone(),
key_id: Some(vector["keyId"].as_str().unwrap().to_string()),
on_missing_key: None,
})
.unwrap();
let plain = cipher
.decrypt(
vector["table"].as_str().unwrap(),
vector["key"].as_str().unwrap(),
vector["field"].as_str().unwrap(),
pair["rustEnvelope"].as_str().unwrap(),
)
.unwrap()
.unwrap();
assert_eq!(plain, vector["value"], "{name}: the recorded envelope no longer opens");
assert_eq!(pair["rustEnvelope"], pair["tsEnvelopeAtSameIv"], "{name}");
}
}