{
"suite": "pith",
"crate": "pith-text",
"format_version": 1,
"generator": "cargo run --bin gen-reference -- gen",
"description": "Hex-exact tokenization, shingling and MinHash vectors for pith-text: canonical UTF-8 bytes, word and shingle counts, 128-word signature folds and Jaccard estimates (IEEE-754 bits, exact) over the committed text fixtures and inline edge inputs.",
"vectors": [
{
"name": "canon-empty",
"input_kind": "inline-hex",
"input_hex": "",
"canonical_sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b",
"canonical_fnv1a64": "af63c74c8601c8dd",
"word_count": 0,
"shingle_count": 0
},
{
"name": "canon-whitespace-only",
"input_kind": "inline-hex",
"input_hex": "2020200a200920",
"canonical_sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b",
"canonical_fnv1a64": "af63c74c8601c8dd",
"word_count": 0,
"shingle_count": 0
},
{
"name": "canon-trailing-crlf",
"input_kind": "inline-hex",
"input_hex": "746578740d0a",
"canonical_sha256": "b9e68e1bea3e5b19ca6b2f98b73a54b73daafaa250484902e09982e07a12e733",
"canonical_fnv1a64": "6baca648789c851c",
"word_count": 1,
"shingle_count": 1
},
{
"name": "canon-mixed-case",
"input_kind": "inline-hex",
"input_hex": "48656c6c6f20574f524c44",
"canonical_sha256": "a948904f2f0f479b8f8197694b30184b0d2ed1c1cd2a1ec0fb85d299a192a447",
"canonical_fnv1a64": "782e1488cd5a68b7",
"word_count": 2,
"shingle_count": 1
},
{
"name": "canon-vietnamese-nfd",
"input_kind": "inline-hex",
"input_hex": "5469c3aacc816e67205669c3aacca374",
"canonical_sha256": "ab30471af32e11a5297b599985b411e89e684cfdbe9c92ec1438af75e0df729f",
"canonical_fnv1a64": "0de652d55ef15669",
"word_count": 2,
"shingle_count": 1
},
{
"name": "canon-turkish-dotted-capital",
"input_kind": "inline-hex",
"input_hex": "c4b05354414e42554c",
"canonical_sha256": "829ae68144ce1a2e3fb9d25e3aaa1f41caf92aa078c0f94e0ae82c5713bcac85",
"canonical_fnv1a64": "f55c4bbdf80f8e62",
"word_count": 1,
"shingle_count": 1
},
{
"name": "canon-multi-paragraph",
"input_kind": "inline-hex",
"input_hex": "6f6e652074776f2074687265650a0a666f757220666976650a",
"canonical_sha256": "fac6668698d325327f1aabdb5e6e4df665fd0dbe22023579e4a46584136711bc",
"canonical_fnv1a64": "ad3b0dc1321badfb",
"word_count": 5,
"shingle_count": 3
},
{
"name": "fixture-doc_a",
"input_kind": "fixture-file",
"input_path": "tests/fixtures/doc_a.txt",
"input_sha256": "c1eb9f012dcb813e212beb49012408fb26b4bf3b086c0c81c56a4a89ec7977c9",
"canonical_sha256": "c1eb9f012dcb813e212beb49012408fb26b4bf3b086c0c81c56a4a89ec7977c9",
"canonical_fnv1a64": "66ca4fe231b3eb8f",
"word_count": 780,
"shingle_count": 778,
"signature_word_0": "00e36ce8dcd483ec",
"signature_word_1": "002b58f9cf81bd61",
"signature_fnv1a64": "1d875bf90953b512",
"signature_sha256": "17e6a7dd743010ae539e2dba3e749e58fd07f869058c850462c97228abffb165"
},
{
"name": "fixture-doc_a_edit",
"input_kind": "fixture-file",
"input_path": "tests/fixtures/doc_a_edit.txt",
"input_sha256": "4593fba5e4513fdc84591ef31deabea139481252abdc71ec2f14de056f22e773",
"canonical_sha256": "4593fba5e4513fdc84591ef31deabea139481252abdc71ec2f14de056f22e773",
"canonical_fnv1a64": "68c24d06a15836f4",
"word_count": 783,
"shingle_count": 781,
"signature_word_0": "00e36ce8dcd483ec",
"signature_word_1": "002b58f9cf81bd61",
"signature_fnv1a64": "015e8413e6a2953e",
"signature_sha256": "26b25e0545e4eafab00612ff5449428a6b6957ce2c007daefb4c8760c7317e1a"
},
{
"name": "fixture-doc_b",
"input_kind": "fixture-file",
"input_path": "tests/fixtures/doc_b.txt",
"input_sha256": "5e3f18ad6bf8f6f2a5ebaf63e747d2b95af712439b80348c241c093b3b06e65d",
"canonical_sha256": "5e3f18ad6bf8f6f2a5ebaf63e747d2b95af712439b80348c241c093b3b06e65d",
"canonical_fnv1a64": "d61de09c6657bd45",
"word_count": 660,
"shingle_count": 658,
"signature_word_0": "01adcecfd7afabe1",
"signature_word_1": "007135ff003b8144",
"signature_fnv1a64": "bd5f19a94edbb1f2",
"signature_sha256": "a569896af3c618610677d363b4a64dd17a21c821de557c50c465b2d676c387e3"
},
{
"name": "fixture-prose",
"input_kind": "fixture-file",
"input_path": "tests/fixtures/prose.txt",
"input_sha256": "754d411b311d9243901a0b45c1fea81368a2b252d906fdcf1b6b882f1d7e16ee",
"canonical_sha256": "754d411b311d9243901a0b45c1fea81368a2b252d906fdcf1b6b882f1d7e16ee",
"canonical_fnv1a64": "2477189016cb64d5",
"word_count": 300,
"shingle_count": 298,
"signature_word_0": "04cf73bf9d43ccef",
"signature_word_1": "005908516233326d",
"signature_fnv1a64": "601da128eecabd2f",
"signature_sha256": "08ea174e093a974ca28d03e88e5bfcb604eda7c39847937ee424135a61107fc1"
},
{
"name": "fixture-punct_tags",
"input_kind": "fixture-file",
"input_path": "tests/fixtures/punct_tags.txt",
"input_sha256": "24538fa0872b9c19f005c765fb3b37215d87ee67b43c2cbabff5aad89c7efbe3",
"canonical_sha256": "013134b64ce1fa89d7cc1e9a021715ffdbfffc3b0d079e905d8ff05129bdc51f",
"canonical_fnv1a64": "a80861ed3cd30d6f",
"word_count": 56,
"shingle_count": 54,
"signature_word_0": "1fa62f5d462e6244",
"signature_word_1": "1c5868a4b35434ab",
"signature_fnv1a64": "eeda536952945be1",
"signature_sha256": "d8c2bac7b5c0ec35555a131b4e1ca6e0431ef74c371e3cbf218f11136dfd6bbf"
},
{
"name": "fixture-short_two",
"input_kind": "fixture-file",
"input_path": "tests/fixtures/short_two.txt",
"input_sha256": "aaf3183a9fd3134e00b0fdcd1d5fde6394526d371b77827185e328c5d13ee42c",
"canonical_sha256": "aaf3183a9fd3134e00b0fdcd1d5fde6394526d371b77827185e328c5d13ee42c",
"canonical_fnv1a64": "1ae5a1a55a0e73c0",
"word_count": 2,
"shingle_count": 1,
"signature_word_0": "894b774d932fb23a",
"signature_word_1": "c838b08f4545f3e7",
"signature_fnv1a64": "15da6abf1818d361",
"signature_sha256": "9675f4b3ef7256ed4289ae8fa5dbabe9b9c6b05a013fb58b2d5c7fe0ba378dd9"
},
{
"name": "fixture-vi_nfc",
"input_kind": "fixture-file",
"input_path": "tests/fixtures/vi_nfc.txt",
"input_sha256": "dafaca378c617539d844564c9f22743845feb1ed11ca959bc161167b82a62562",
"canonical_sha256": "3051a8d56b755d23aa3ad69ac833eb8f40da7b840900434c38fb31f130ecc83e",
"canonical_fnv1a64": "d3fcf231bcfcdd15",
"word_count": 27,
"shingle_count": 25,
"signature_word_0": "1665173b7a6441c5",
"signature_word_1": "27fe74da8b9e7412",
"signature_fnv1a64": "724766d7b8611548",
"signature_sha256": "4c110fcee0d0ca8bfc0a2864594e422b44a0b8ebec33188f162eca3d354580d1"
},
{
"name": "fixture-vi_nfd",
"input_kind": "fixture-file",
"input_path": "tests/fixtures/vi_nfd.txt",
"input_sha256": "b6d2a058afc1e9e0ce834a1f97bdae5fdd93f74f9ffeae2bc5d8d9c5d45e98b2",
"canonical_sha256": "3051a8d56b755d23aa3ad69ac833eb8f40da7b840900434c38fb31f130ecc83e",
"canonical_fnv1a64": "d3fcf231bcfcdd15",
"word_count": 27,
"shingle_count": 25,
"signature_word_0": "1665173b7a6441c5",
"signature_word_1": "27fe74da8b9e7412",
"signature_fnv1a64": "724766d7b8611548",
"signature_sha256": "4c110fcee0d0ca8bfc0a2864594e422b44a0b8ebec33188f162eca3d354580d1"
},
{
"name": "signature-alpha-beta-gamma",
"input_kind": "inline-hex",
"input_hex": "616c70686120626574612067616d6d61",
"signature_words_hex": [
"5409aadb22bb8479",
"b2e6ff4edebf53c6",
"102ac123bc90828c",
"e0942f929a57699c",
"93d57ad5a7d9fd33",
"b276e7db98dfa4b5",
"1c2fa1e0a927782a",
"5f0f43e68f2c37b7",
"321c735a76b57081",
"4a943bea9d33cebe",
"f431aa9a165a7b99",
"07c8e3c76230616e",
"52b83ea5aa947612",
"b260f1ae68de0ddd",
"50b0c63be5a45df3",
"0392584ee2eae53e",
"644de82fff51474c",
"6aa93d555f52621c",
"e951a1f45cc3bfcc",
"9701a30634b1162e",
"8e15335566d5c509",
"da5ee847044d8d12",
"88a8d4b2784cd2c2",
"221691dcd6ea72ac",
"15bc90c1e9682260",
"078533db9c1aa917",
"f068fe75b629e507",
"b621f56d1b905d02",
"55953dc2842a2f1e",
"81c342012309e7aa",
"c9792bdedbf05dbd",
"a20a4cc72ba8e14c",
"e240972888809a57",
"8a1fe48cfdc451c5",
"a03ace33a87cb096",
"78b295854d58734f",
"92e8a5d9725259a4",
"467b7affc8fdf45b",
"d711bc680d2d808a",
"ab05c3dfe87073be",
"cc60a041293a88c1",
"ce154f2bcb7b6047",
"65713243f39adaf9",
"07cfc1ce2fbca771",
"cc04dad267468a86",
"7d42c63f605a3a8a",
"f364b5c5e980e83e",
"e66261e5fce4f7fe",
"deba29d572c8db48",
"da49501019ed4be2",
"cd1291b7a4d1e51e",
"6486bcf2e8625ab3",
"5d2f6dbbb1836c2b",
"e1403c08505b35cf",
"5885dc76023decc4",
"fa781c144fd6388f",
"9b0013447a3943ca",
"a412d32cf4d86347",
"0b4a20853928352a",
"2ea0cdce88006f0d",
"b0210a155e190048",
"d79bea640bcd5a60",
"ce3430b4bf8a38ad",
"e4df26e05e5fa19a",
"bc6d7f84bad1c81a",
"041086bd0b0f067e",
"d031c1f52ce10a41",
"328b4f5ab251c6cb",
"24329a90be0fdf5d",
"7a50d025bbf00d54",
"2301d6651860d647",
"a846d7235c533e3c",
"b74d8508e00ebbbb",
"2a0ed33cd467e36c",
"75b0c7131e9ef306",
"028a8cc6cdfbe97d",
"d89590c250010d32",
"7ef6d87898404aa9",
"a6215d61d5717669",
"8e8b0a245a745e0a",
"0bd7b867a352863d",
"160ce4e7e164e119",
"4194160aa23a0e75",
"6ea4af6e5971547c",
"049fa8bf6a64950e",
"de335af325294351",
"5f343bd8a43fe817",
"6295b89a3281403d",
"cd43f91a2f1cc06f",
"a7fcba9f0ab2a7c5",
"4a083b9f1e15729a",
"8004d6f9ed73caac",
"b261fca5f14e9757",
"f3a4bf091cbdde44",
"18a5de339820414d",
"3b3d580c90bd2840",
"b13c1d69433f94b1",
"56036a6566994ec0",
"6d5d10962287793f",
"3542f6c837137348",
"2a470233f14079b3",
"9aa0ab530f6922b5",
"ea5f4c45b326369b",
"fd8b2813bbd35b04",
"db405a37020746a9",
"b9d5349414b97184",
"919b0e4fadc5425d",
"5fedecde2c7a6f55",
"a1775ff616bbec80",
"d2ec56c37900892c",
"6a426affa1a9e53b",
"79529e1092cfca1c",
"4a5f808195d63d5e",
"b72634e7ed5ead05",
"a810bd13bb6aa48f",
"f3ce1053907a2c1e",
"d0842838be063f18",
"0a0c68adff99c55a",
"8599f456c911957f",
"5deb1d0fb2f9a11b",
"d45d92023c587079",
"0249899b62100c72",
"c370f0eabee06e63",
"e412a0eb9a69ba16",
"a1d2cf1f2098cdb9",
"d352b47e8e2a711f",
"faaa04e04b142c16",
"8d0301424d59a445"
],
"signature_fnv1a64": "46f8ab1a50c61813",
"signature_sha256": "5a3e4c4a48d55a405faced9c28d4bf12b78af5661102eb2ac3dcab843485d8df"
},
{
"name": "signature-empty-sentinel",
"input_kind": "inline-hex",
"input_hex": "",
"signature_word_0": "ffffffffffffffff",
"signature_word_1": "ffffffffffffffff",
"signature_fnv1a64": "89bfa3a928539725",
"signature_sha256": "5f4ecdb7b71c3e403983fe405cddcdc2f2576b655fdb3e80d94a6f7c32e58bc2"
},
{
"name": "jaccard-near-duplicate",
"a": "fixture:tests/fixtures/doc_a.txt",
"b": "fixture:tests/fixtures/doc_a_edit.txt",
"value_kind": "f64-ieee754-bits-hex",
"policy": "exact",
"value_bits": "3fef800000000000"
},
{
"name": "jaccard-unrelated",
"a": "fixture:tests/fixtures/doc_a.txt",
"b": "fixture:tests/fixtures/doc_b.txt",
"value_kind": "f64-ieee754-bits-hex",
"policy": "exact",
"value_bits": "0000000000000000"
},
{
"name": "jaccard-identical",
"a": "fixture:tests/fixtures/prose.txt",
"b": "fixture:tests/fixtures/prose.txt",
"value_kind": "f64-ieee754-bits-hex",
"policy": "exact",
"value_bits": "3ff0000000000000"
},
{
"name": "jaccard-cross-language",
"a": "fixture:tests/fixtures/vi_nfc.txt",
"b": "fixture:tests/fixtures/doc_b.txt",
"value_kind": "f64-ieee754-bits-hex",
"policy": "exact",
"value_bits": "0000000000000000"
},
{
"name": "jaccard-against-empty",
"a": "fixture:tests/fixtures/doc_a.txt",
"b": "empty-signature",
"value_kind": "f64-ieee754-bits-hex",
"policy": "exact",
"value_bits": "3ff0000000000000"
}
]
}