use frankensearch_core::generation::{
CanonicalDocsetV1, ExactComponentReceiptV1, GenerationAuthorityErrorV1,
GenerationComponentReceiptV1, GenerationComponentRole, SourceCheckpointV1,
};
use sha2::{Digest, Sha256};
use crate::FsviV2Witness;
use crate::native_hnsw::NativeHnswGenerationReceiptV2;
fn sha256_from_hex(
label: &'static str,
text: &str,
) -> Result<[u8; 32], GenerationAuthorityErrorV1> {
let bytes = text.as_bytes();
if bytes.len() != 64 || !bytes.iter().all(u8::is_ascii_hexdigit) {
return Err(GenerationAuthorityErrorV1::InvalidField { field: label });
}
let mut digest = [0_u8; 32];
for (index, slot) in digest.iter_mut().enumerate() {
let pair = &text[index * 2..index * 2 + 2];
*slot = u8::from_str_radix(pair, 16)
.map_err(|_| GenerationAuthorityErrorV1::InvalidField { field: label })?;
}
Ok(digest)
}
#[cfg(test)]
fn canonical_docset_digest<I, S>(
ordered_live_documents: I,
) -> Result<[u8; 32], GenerationAuthorityErrorV1>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
Ok(CanonicalDocsetV1::from_ordered_live_documents(ordered_live_documents)?.digest())
}
fn authenticated_docset<I, S>(
ordered_live_documents: I,
expected_fsvi_digest: [u8; 32],
digest_field: &'static str,
) -> Result<(CanonicalDocsetV1, u64), GenerationAuthorityErrorV1>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
let documents = ordered_live_documents
.into_iter()
.map(Into::into)
.collect::<Vec<String>>();
let live_document_count =
u64::try_from(documents.len()).map_err(|_| GenerationAuthorityErrorV1::InvalidField {
field: "component_receipt.live_document_count",
})?;
let mut fsvi_hasher = Sha256::new();
crate::update_digest_domain(&mut fsvi_hasher, crate::ORDERED_DOCSET_DIGEST_DOMAIN);
fsvi_hasher.update(live_document_count.to_be_bytes());
for document in &documents {
let byte_len = u64::try_from(document.len()).map_err(|_| {
GenerationAuthorityErrorV1::InvalidField {
field: "component_receipt.docset.document_id",
}
})?;
fsvi_hasher.update(byte_len.to_be_bytes());
fsvi_hasher.update(document.as_bytes());
}
let observed_fsvi_digest: [u8; 32] = fsvi_hasher.finalize().into();
let canonical = CanonicalDocsetV1::from_ordered_live_documents(documents)?;
if observed_fsvi_digest != expected_fsvi_digest {
return Err(GenerationAuthorityErrorV1::InvalidField {
field: digest_field,
});
}
Ok((canonical, live_document_count))
}
pub fn vector_component_receipt<I, S>(
witness: &FsviV2Witness,
ordered_live_documents: I,
source_checkpoint: SourceCheckpointV1,
) -> Result<ExactComponentReceiptV1, GenerationAuthorityErrorV1>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
let (docset, live_document_count) = authenticated_docset(
ordered_live_documents,
witness.ordered_live_docset_digest,
"component_receipt.vector.ordered_live_docset_digest",
)?;
if live_document_count != witness.live_count {
return Err(GenerationAuthorityErrorV1::InvalidField {
field: "component_receipt.vector.live_document_count",
});
}
let receipt = ExactComponentReceiptV1 {
role: GenerationComponentRole::Vector,
bytes: GenerationComponentReceiptV1 {
byte_len: witness.byte_len,
sha256: witness.whole_image_sha256,
},
docset_digest: docset.digest(),
live_document_count,
source_checkpoint: source_checkpoint.to_bytes(),
};
receipt.validate()?;
Ok(receipt)
}
pub fn ann_component_receipt<I, S>(
receipt: &NativeHnswGenerationReceiptV2,
ordered_live_documents: I,
source_checkpoint: SourceCheckpointV1,
) -> Result<ExactComponentReceiptV1, GenerationAuthorityErrorV1>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
receipt
.validate()
.map_err(|_| GenerationAuthorityErrorV1::InvalidField {
field: "component_receipt.ann.native_receipt",
})?;
let expected_fsvi_digest = sha256_from_hex(
"ordered_live_docset_digest",
&receipt.ordered_live_docset_digest,
)?;
let (docset, live_document_count) = authenticated_docset(
ordered_live_documents,
expected_fsvi_digest,
"component_receipt.ann.ordered_live_docset_digest",
)?;
let component = ExactComponentReceiptV1 {
role: GenerationComponentRole::Ann,
bytes: GenerationComponentReceiptV1 {
byte_len: receipt.graph_byte_len,
sha256: sha256_from_hex("graph_sha256", &receipt.graph_sha256)?,
},
docset_digest: docset.digest(),
live_document_count,
source_checkpoint: source_checkpoint.to_bytes(),
};
component.validate()?;
Ok(component)
}
#[cfg(test)]
mod tests {
use super::canonical_docset_digest;
use frankensearch_core::generation::{
CanonicalDocsetV1, CommitRange, GenerationAuthorityErrorV1, SourceCheckpointV1,
};
use sha2::{Digest, Sha256};
fn fsvi_domain_digest(documents: &[&str]) -> [u8; 32] {
const ORDERED_DOCSET_DIGEST_DOMAIN: &[u8] = b"frankensearch.fsvi-v2.ordered-live-docset.v1";
let mut hasher = Sha256::new();
hasher.update(
u64::try_from(ORDERED_DOCSET_DIGEST_DOMAIN.len())
.unwrap_or(u64::MAX)
.to_be_bytes(),
);
hasher.update(ORDERED_DOCSET_DIGEST_DOMAIN);
hasher.update((documents.len() as u64).to_be_bytes());
for document in documents {
hasher.update((document.len() as u64).to_be_bytes());
hasher.update(document.as_bytes());
}
hasher.finalize().into()
}
#[test]
fn adapter_docset_digest_is_canonical_not_fsvi_domain() {
let documents = ["doc-a", "doc-b", "doc-c"];
let canonical = CanonicalDocsetV1::from_ordered_live_documents(documents)
.expect("canonical docset")
.digest();
let fsvi = fsvi_domain_digest(&documents);
assert_ne!(
canonical, fsvi,
"the canonical and FSVI-domain digests must differ over the same ordered \
identifiers; if they ever coincide, copying the witness value would silently \
become correct and this adapter's whole reason for existing would be invisible"
);
let emitted = canonical_docset_digest(documents).expect("adapter digest");
assert_eq!(
emitted, canonical,
"the adapter must emit the CANONICAL digest"
);
assert_ne!(
emitted, fsvi,
"the adapter must never emit the FSVI-domain digest"
);
}
#[test]
fn canonical_digest_is_order_sensitive() {
let forward = canonical_docset_digest(["doc-a", "doc-b"]).expect("forward");
let reversed = canonical_docset_digest(["doc-b", "doc-a"]).expect("reversed");
assert_ne!(
forward, reversed,
"a reordered docset must not produce the same canonical digest"
);
}
#[test]
fn duplicate_identifiers_are_rejected_not_deduplicated() {
assert!(
canonical_docset_digest(["doc-a", "doc-a"]).is_err(),
"a duplicate identifier must be a typed error"
);
}
use frankensearch_core::generation::{
ComponentJoinErrorV1, ExactComponentReceiptV1, ExactGenerationComponentsV1,
GenerationComponentReceiptV1, GenerationComponentRole,
};
use super::{ann_component_receipt, sha256_from_hex, vector_component_receipt};
use crate::native_hnsw::{NativeHnswGenerationReceiptV2, NativeHnswParamsIdentityV1};
use crate::{FsviV2Witness, Quantization};
use frankensearch_core::generation::ArtifactGenerationIdentityV1;
const DOCS: [&str; 3] = ["doc-a", "doc-b", "doc-c"];
const RANGE: CommitRange = CommitRange { low: 1, high: 100 };
const OTHER_RANGE: CommitRange = CommitRange { low: 1, high: 101 };
fn checkpoint() -> SourceCheckpointV1 {
SourceCheckpointV1::derive(&RANGE)
}
const WITNESS_IMAGE_SHA: [u8; 32] = [0xA1; 32];
const WITNESS_CONTENT_DIGEST: [u8; 32] = [0xA3; 32];
fn witness_fixture() -> FsviV2Witness {
FsviV2Witness {
schema_version: 1,
fsvi_version: 2,
byte_len: 8_192,
whole_image_sha256: WITNESS_IMAGE_SHA,
generation: ArtifactGenerationIdentityV1::new(7, [0x4d; 16])
.expect("valid test generation"),
identity_bundle_fingerprint: [0xB1; 32],
space_fingerprint: [0xB2; 32],
producer_fingerprint: [0xB3; 32],
input_fingerprint: [0xB4; 32],
storage_fingerprint: [0xB5; 32],
generation_fingerprint: [0xB6; 32],
ordered_live_docset_digest: fsvi_domain_digest(&DOCS),
vector_content_digest: WITNESS_CONTENT_DIGEST,
dimension: 4,
quantization: Quantization::F32,
record_count: 5,
live_count: DOCS.len() as u64,
tombstone_count: 2,
}
}
fn hex32(byte: u8) -> String {
use std::fmt::Write as _;
let mut text = String::with_capacity(64);
for _ in 0..32 {
write!(text, "{byte:02x}").expect("writing to a String cannot fail");
}
text
}
fn hex_digest(digest: [u8; 32]) -> String {
use std::fmt::Write as _;
let mut text = String::with_capacity(64);
for byte in digest {
write!(text, "{byte:02x}").expect("writing to a String cannot fail");
}
text
}
fn ann_receipt_fixture() -> NativeHnswGenerationReceiptV2 {
let artifact_generation =
ArtifactGenerationIdentityV1::new(7, [0x4d; 16]).expect("valid test generation");
let mut receipt = NativeHnswGenerationReceiptV2 {
schema_version: 2,
artifact_generation,
artifact_generation_fingerprint: artifact_generation.fingerprint(),
embedding_identity_fingerprint: hex32(0xC2),
embedding_space_fingerprint: hex32(0xC3),
embedding_producer_fingerprint: hex32(0xC4),
embedding_input_fingerprint: hex32(0xC5),
vector_storage_fingerprint: hex32(0xC6),
vector_content_digest: hex32(0xC7),
ordered_live_docset_digest: hex_digest(fsvi_domain_digest(&DOCS)),
fsvi_whole_image_sha256: hex32(0xA1),
fsvi_physical_row_count: 5,
graph_basename: "fast.fshnsw".to_owned(),
graph_byte_len: 4_096,
graph_sha256: hex32(0xD1),
native_format_version: 1,
params: NativeHnswParamsIdentityV1 {
m: 16,
m0: 32,
ef_construction: 64,
ef_search: 32,
},
seed: 42,
point_count: 5,
entry_point: Some(0),
max_level: 3,
payload_crc32: 0x1234_5678,
header_crc32: 0x8765_4321,
topology_sha256: hex32(0xD2),
receipt_sha256: String::new(),
};
receipt.seal_for_test().expect("seal test ANN receipt");
receipt.validate().expect("test ANN receipt validates");
receipt
}
#[test]
fn the_vector_adapter_maps_every_field_from_the_witness() {
let witness = witness_fixture();
let receipt = vector_component_receipt(&witness, DOCS, checkpoint())
.expect("a valid witness produces a vector receipt");
assert_eq!(receipt.role, GenerationComponentRole::Vector);
assert_eq!(receipt.bytes.byte_len, witness.byte_len);
assert_eq!(receipt.bytes.sha256, witness.whole_image_sha256);
assert_ne!(
receipt.bytes.sha256, WITNESS_CONTENT_DIGEST,
"the component bytes are the whole IMAGE, not the vector content digest"
);
assert_eq!(
receipt.live_document_count, witness.live_count,
"live count, not record_count"
);
assert_ne!(receipt.live_document_count, witness.record_count);
assert_eq!(receipt.source_checkpoint, checkpoint().to_bytes());
assert_eq!(
receipt.docset_digest,
CanonicalDocsetV1::from_ordered_live_documents(DOCS)
.expect("canonical")
.digest()
);
assert_ne!(
receipt.docset_digest, witness.ordered_live_docset_digest,
"re-labelling the witness's fsvi-domain digest would be the whole bug"
);
}
#[test]
fn the_ann_adapter_binds_the_graph_not_the_fsvi_image() {
let ann = ann_receipt_fixture();
let receipt = ann_component_receipt(&ann, DOCS, checkpoint())
.expect("a valid ANN receipt produces an ANN component");
assert_eq!(receipt.role, GenerationComponentRole::Ann);
assert_eq!(receipt.bytes.byte_len, ann.graph_byte_len);
assert_eq!(
receipt.bytes.sha256,
sha256_from_hex("graph_sha256", &ann.graph_sha256).expect("fixture hex")
);
assert_ne!(
receipt.bytes.byte_len, 8_192,
"that is the FSVI image length"
);
assert_ne!(
receipt.bytes.sha256,
sha256_from_hex("fsvi", &ann.fsvi_whole_image_sha256).expect("fixture hex"),
"the ANN component must not bind the FSVI whole-image digest"
);
assert_eq!(receipt.live_document_count, DOCS.len() as u64);
assert_ne!(receipt.live_document_count, ann.point_count);
assert_eq!(receipt.source_checkpoint, checkpoint().to_bytes());
assert_ne!(
receipt.docset_digest,
sha256_from_hex("docset", &ann.ordered_live_docset_digest).expect("fixture hex"),
"the ANN adapter must recompute the canonical digest too"
);
}
fn metadata_component(
docset_digest: [u8; 32],
checkpoint: SourceCheckpointV1,
) -> ExactComponentReceiptV1 {
ExactComponentReceiptV1 {
role: GenerationComponentRole::Metadata,
bytes: GenerationComponentReceiptV1 {
byte_len: 512,
sha256: [0xE1; 32],
},
docset_digest,
live_document_count: DOCS.len() as u64,
source_checkpoint: checkpoint.to_bytes(),
}
}
fn lexical_component(
docset_digest: [u8; 32],
checkpoint: SourceCheckpointV1,
) -> ExactComponentReceiptV1 {
ExactComponentReceiptV1 {
role: GenerationComponentRole::Lexical,
bytes: GenerationComponentReceiptV1 {
byte_len: 1_024,
sha256: [0xE2; 32],
},
docset_digest,
live_document_count: DOCS.len() as u64,
source_checkpoint: checkpoint.to_bytes(),
}
}
#[test]
fn adapter_produced_receipts_admit_as_one_generation() {
let vector = vector_component_receipt(&witness_fixture(), DOCS, checkpoint())
.expect("vector receipt");
let ann =
ann_component_receipt(&ann_receipt_fixture(), DOCS, checkpoint()).expect("ann receipt");
let canonical = vector.docset_digest;
let admitted = ExactGenerationComponentsV1::admit(
vector.clone(),
lexical_component(canonical, checkpoint()),
Some(ann.clone()),
metadata_component(canonical, checkpoint()),
)
.expect("adapter-produced receipts describe one generation");
assert_eq!(admitted.vector(), &vector);
assert_eq!(admitted.ann(), Some(&ann));
assert_eq!(admitted.docset_digest(), canonical);
assert!(admitted.has_ann());
ExactGenerationComponentsV1::admit(
vector,
lexical_component(canonical, checkpoint()),
None,
metadata_component(canonical, checkpoint()),
)
.expect("a generation without an ANN sidecar is still exact");
}
#[test]
fn adapters_reject_docsets_not_authenticated_by_the_producer() {
let reordered = ["doc-a", "doc-c", "doc-b"];
let substituted = ["doc-a", "doc-b", "doc-z"];
assert_eq!(reordered.len(), DOCS.len());
assert_eq!(substituted.len(), DOCS.len());
for documents in [reordered, substituted] {
assert!(
vector_component_receipt(&witness_fixture(), documents, checkpoint()).is_err(),
"vector adapter accepted unwitnessed documents: {documents:?}"
);
assert!(
ann_component_receipt(&ann_receipt_fixture(), documents, checkpoint()).is_err(),
"ANN adapter accepted unwitnessed documents: {documents:?}"
);
}
vector_component_receipt(&witness_fixture(), DOCS, checkpoint())
.expect("the witnessed vector docset remains valid");
ann_component_receipt(&ann_receipt_fixture(), DOCS, checkpoint())
.expect("the witnessed ANN docset remains valid");
}
#[test]
fn vector_adapter_rejects_a_witness_count_that_contradicts_its_digest() {
let mut witness = witness_fixture();
witness.live_count += 1;
assert!(matches!(
vector_component_receipt(&witness, DOCS, checkpoint()),
Err(GenerationAuthorityErrorV1::InvalidField {
field: "component_receipt.vector.live_document_count"
})
));
vector_component_receipt(&witness_fixture(), DOCS, checkpoint())
.expect("the consistent witness remains valid");
}
#[test]
fn ann_adapter_rejects_a_receipt_with_a_stale_internal_seal() {
let mut receipt = ann_receipt_fixture();
receipt.graph_byte_len += 1;
assert!(
receipt.validate().is_err(),
"the mutation control must invalidate the receipt body seal"
);
assert!(matches!(
ann_component_receipt(&receipt, DOCS, checkpoint()),
Err(GenerationAuthorityErrorV1::InvalidField {
field: "component_receipt.ann.native_receipt"
})
));
ann_component_receipt(&ann_receipt_fixture(), DOCS, checkpoint())
.expect("the sealed control receipt remains admissible");
}
#[test]
fn an_adapter_given_a_different_checkpoint_rejects_on_its_own_role() {
let vector = vector_component_receipt(&witness_fixture(), DOCS, checkpoint())
.expect("vector receipt");
let canonical = vector.docset_digest;
let other_checkpoint = SourceCheckpointV1::derive(&OTHER_RANGE);
assert_ne!(other_checkpoint, checkpoint());
let ann = ann_component_receipt(&ann_receipt_fixture(), DOCS, other_checkpoint)
.expect("ann receipt at another checkpoint");
assert_eq!(
ann.docset_digest, canonical,
"documents must be identical, or this is docset drift instead"
);
let observed = ExactGenerationComponentsV1::admit(
vector,
lexical_component(canonical, checkpoint()),
Some(ann),
metadata_component(canonical, checkpoint()),
);
assert!(
matches!(
observed,
Err(ComponentJoinErrorV1::CheckpointDrift { role: "ann" })
),
"observed {observed:?}"
);
}
#[test]
fn malformed_hex_digests_are_typed_errors_not_silent_zeros() {
let valid = hex32(0xD1);
sha256_from_hex("control", &valid).expect("64 lowercase hex digits decode");
for (label, text) in [
("empty", String::new()),
("too short", valid[..62].to_owned()),
("too long", format!("{valid}00")),
("non-hex", "z".repeat(64)),
("spaces", " ".repeat(64)),
] {
assert!(
sha256_from_hex("graph_sha256", &text).is_err(),
"{label} must be a typed error"
);
}
assert_eq!(
sha256_from_hex("upper", &valid.to_uppercase()).expect("uppercase decodes"),
sha256_from_hex("lower", &valid).expect("lowercase decodes")
);
let zeros = "0".repeat(64);
assert_eq!(
sha256_from_hex("zeros", &zeros).expect("all-zero hex is well formed"),
[0_u8; 32]
);
let mut ann = ann_receipt_fixture();
ann.graph_sha256 = zeros;
assert!(
ann_component_receipt(&ann, DOCS, checkpoint()).is_err(),
"an all-zero graph digest cannot identify real bytes"
);
}
#[test]
fn neither_adapter_produces_a_receipt_from_an_invalid_docset() {
let witness = witness_fixture();
let ann = ann_receipt_fixture();
assert!(vector_component_receipt(&witness, ["doc-a", "doc-a"], checkpoint()).is_err());
assert!(ann_component_receipt(&ann, ["doc-a", "doc-a"], checkpoint()).is_err());
assert!(vector_component_receipt(&witness, ["doc-a", ""], checkpoint()).is_err());
assert!(ann_component_receipt(&ann, ["", "doc-b"], checkpoint()).is_err());
vector_component_receipt(&witness, DOCS, checkpoint()).expect("valid docset");
ann_component_receipt(&ann, DOCS, checkpoint()).expect("valid docset");
}
#[test]
fn every_non_anchor_role_is_blamed_for_its_own_checkpoint_drift() {
let anchor = vector_component_receipt(&witness_fixture(), DOCS, checkpoint())
.expect("anchor vector receipt");
let canonical = anchor.docset_digest;
let other = SourceCheckpointV1::derive(&OTHER_RANGE);
assert_ne!(other, checkpoint(), "the fixture must move the checkpoint");
let observed = ExactGenerationComponentsV1::admit(
anchor.clone(),
lexical_component(canonical, checkpoint()),
Some(
ann_component_receipt(&ann_receipt_fixture(), DOCS, other)
.expect("ann receipt at another checkpoint"),
),
metadata_component(canonical, checkpoint()),
);
assert!(
matches!(
observed,
Err(ComponentJoinErrorV1::CheckpointDrift { role: "ann" })
),
"ann drift must name ann: {observed:?}"
);
let observed = ExactGenerationComponentsV1::admit(
anchor.clone(),
lexical_component(canonical, other),
None,
metadata_component(canonical, checkpoint()),
);
assert!(
matches!(
observed,
Err(ComponentJoinErrorV1::CheckpointDrift { role: "lexical" })
),
"lexical drift must name lexical: {observed:?}"
);
let observed = ExactGenerationComponentsV1::admit(
anchor.clone(),
lexical_component(canonical, checkpoint()),
None,
metadata_component(canonical, other),
);
assert!(
matches!(
observed,
Err(ComponentJoinErrorV1::CheckpointDrift { role: "metadata" })
),
"metadata drift must name metadata: {observed:?}"
);
ExactGenerationComponentsV1::admit(
anchor,
lexical_component(canonical, checkpoint()),
Some(
ann_component_receipt(&ann_receipt_fixture(), DOCS, checkpoint())
.expect("ann receipt"),
),
metadata_component(canonical, checkpoint()),
)
.expect("one derived checkpoint across all four roles admits");
}
#[test]
fn a_drifted_anchor_is_reported_against_the_roles_that_disagree_with_it() {
let drifted_anchor = vector_component_receipt(
&witness_fixture(),
DOCS,
SourceCheckpointV1::derive(&OTHER_RANGE),
)
.expect("vector receipt at another checkpoint");
let canonical = drifted_anchor.docset_digest;
let observed = ExactGenerationComponentsV1::admit(
drifted_anchor,
lexical_component(canonical, checkpoint()),
None,
metadata_component(canonical, checkpoint()),
);
assert!(
matches!(
observed,
Err(ComponentJoinErrorV1::CheckpointDrift { role: "lexical" })
),
"a drifted anchor surfaces as disagreement from the first checked \
mandatory role, not as a vector error: {observed:?}"
);
}
#[test]
fn no_commit_range_derives_the_zero_placeholder_checkpoint() {
for range in [
CommitRange { low: 0, high: 0 },
CommitRange { low: 0, high: 1 },
CommitRange { low: 1, high: 1 },
CommitRange { low: 1, high: 100 },
CommitRange {
low: u64::MAX,
high: u64::MAX,
},
CommitRange { low: 5, high: 1 },
] {
assert_ne!(
SourceCheckpointV1::derive(&range).to_bytes(),
[0; 32],
"{range:?} derived the all-zero placeholder"
);
}
assert_ne!(
SourceCheckpointV1::derive(&CommitRange { low: 0, high: 1 }),
SourceCheckpointV1::derive(&CommitRange { low: 1, high: 0 })
);
}
}