# SPDX-License-Identifier: MIT OR Apache-2.0
#
# Runs the cross-validation suites that back anamnesis's correctness claims,
# and says plainly what they do and do not prove.
#
# Why this script exists: `cargo test --all-features` already runs everything.
# What a reader cannot know from the outside is *which* of the 30-odd test
# binaries constitute the "bit-exact, 0 ULP" claim. This selects exactly those
# and reports the coverage.
#
# Usage: .\scripts\verify-claims.ps1
$ErrorActionPreference = "Stop"
Set-Location (Join-Path $PSScriptRoot "..")
if (-not (Test-Path "tests/fixtures")) {
Write-Error @"
tests/fixtures/ is missing.
You are probably in a crate unpacked from crates.io, which excludes tests/
to keep the published artefact at 0.60 MiB instead of 4.8 MiB.
Get the corpus from the matching GitHub Release (the "Source code (tar.gz)"
asset carries tests/ verbatim), or clone the repository:
https://github.com/mi-for-the-rust-of-us/anamnesis/releases
"@
exit 1
}
if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) {
Write-Error "cargo not found. Install Rust from https://rustup.rs"
exit 1
}
Write-Host "anamnesis correctness verification"
Write-Host "=================================="
Write-Host ""
$suites = @(
@{ Target = "cross_validation_safetensors"; What = "FP8 E4M3 (fine-grained / per-channel / per-tensor), BF16 + F32"; Oracle = "PyTorch's own fp8 cast" }
@{ Target = "cross_validation_gptq"; What = "GPTQ INT4 + INT8, group-wise, g_idx, BF16 + F32"; Oracle = "GPTQModel dequantize_weight" }
@{ Target = "cross_validation_awq"; What = "AWQ INT4, per-group, BF16 + F32"; Oracle = "AutoAWQ unpack_awq + reverse_awq_order" }
@{ Target = "cross_validation_bnb"; What = "BitsAndBytes NF4 / FP4 / INT8, BF16 + F32"; Oracle = "bitsandbytes dequantize_4bit" }
@{ Target = "cross_validation_bnb_encode"; What = "BitsAndBytes NF4 encode (byte-exact on disk)"; Oracle = "bitsandbytes on-disk bytes" }
@{ Target = "cross_validation_gguf"; What = "all 22 GGUF block-quant kernels, BF16 + F32"; Oracle = "gguf-py (mirrors ggml-quants.c)" }
@{ Target = "cross_validation_ollama"; What = "GGUF Q8_0 from a real Ollama blob"; Oracle = "gguf-py" }
@{ Target = "cross_validation_npz"; What = "NPZ / NPY parsing"; Oracle = "NumPy" }
@{ Target = "cross_validation_pth"; What = "PyTorch .pth (pickle + tensor recovery)"; Oracle = "torch.load" }
@{ Target = "cross_validation_convert"; What = "format-conversion pipeline"; Oracle = "per-format references" }
@{ Target = "cross_validation"; What = "end-to-end dequantisation"; Oracle = "per-format references" }
)
$failed = 0
foreach ($s in $suites) {
Write-Host ("{0,-34} {1}" -f $s.Target, $s.What)
Write-Host ("{0,-34} reference: {1}" -f "", $s.Oracle)
$log = cargo test --release --all-features --test $s.Target -- --quiet 2>&1
if ($LASTEXITCODE -eq 0) {
# Count from the summary line, not from `^test .* ok$`: `--quiet` prints
# a dot per test and never those lines, so the old pattern matched
# nothing and every suite reported "PASS 0 tests". A zero count is now
# a FAILURE rather than a silent pass, because "the binary compiled and
# ran no tests" and "22 kernels verified" must not look identical in a
# script whose entire job is substantiating a correctness claim.
$count = 0
foreach ($m in ($log | Select-String -Pattern 'test result: ok\. (\d+) passed')) {
$count += [int]$m.Matches[0].Groups[1].Value
}
if ($count -eq 0) {
Write-Host ("{0,-34} FAIL ran 0 tests (suite filtered out or empty)" -f "") -ForegroundColor Red
$failed = 1
} else {
Write-Host ("{0,-34} PASS {1} tests" -f "", $count) -ForegroundColor Green
}
} else {
Write-Host ("{0,-34} FAIL" -f "") -ForegroundColor Red
$log | Select-Object -Last 30 | ForEach-Object { Write-Host $_ }
$failed = 1
}
Write-Host ""
}
Write-Host @"
------------------------------------------------------------------------
What this proves, and what it does not
PROVES: anamnesis's dequantisation output matches goldens committed to this
repository, byte for byte. Since v0.7.4 that includes a full-width F32
comparison with no tolerance at all for *every* dequantising family: all 22
GGUF kernels, plus FP8, GPTQ, AWQ and BitsAndBytes. That matters because a
BF16 comparison discards 16 mantissa bits, and it is what caught a 1-ULP
defect in the BnB INT8 kernel that five releases of BF16 testing had passed.
DOES NOT PROVE: that those goldens are themselves correct. They were
generated by each format's own canonical library (see the "reference" line
per suite), but this run does not re-derive them.
To re-derive the goldens from the references, which is the stronger check:
pip install gguf numpy
python tests/fixtures/gguf_reference/generate_gguf.py --upgrade
That rebuilds every GGUF fixture from its own raw quantised bytes, needs no
model download, and refuses to proceed unless the regenerated BF16 golden
reproduces the committed one byte for byte. The other families' generators
need their respective libraries (torch, bitsandbytes, autoawq, gptqmodel)
and the source models; see tests/fixtures/*/ for each generator.
Fixture provenance is a first-class concern here: hand-rolled reference
reimplementations are banned from the generators, because three bugs once
shipped green behind circular fixtures (v0.6.4).
------------------------------------------------------------------------
"@
exit $failed