#!/usr/bin/env bash
# SPDX-License-Identifier: MIT OR Apache-2.0
#
# Runs the cross-validation suites that back anamnesis's correctness claims,
# and says plainly what they do and do not prove.
#
# Why this script exists: `cargo test --all-features` already runs everything.
# What a reader cannot know from the outside is *which* of the 30-odd test
# binaries constitute the "bit-exact, 0 ULP" claim. This selects exactly those
# and reports the coverage.
#
# Usage:  ./scripts/verify-claims.sh
set -euo pipefail

cd "$(dirname "$0")/.."

if [ ! -d tests/fixtures ]; then
  cat >&2 <<'MISSING'
error: tests/fixtures/ is missing.

You are probably in a crate unpacked from crates.io, which excludes tests/
to keep the published artefact at 0.60 MiB instead of 4.8 MiB.

Get the corpus from the matching GitHub Release (the "Source code (tar.gz)"
asset carries tests/ verbatim), or clone the repository:

  https://github.com/mi-for-the-rust-of-us/anamnesis/releases
MISSING
  exit 1
fi

command -v cargo >/dev/null 2>&1 || {
  echo "error: cargo not found. Install Rust from https://rustup.rs" >&2
  exit 1
}

echo "anamnesis correctness verification"
echo "=================================="
echo

SUITES=(
  "cross_validation_safetensors:FP8 E4M3 (fine-grained / per-channel / per-tensor), BF16 + F32:PyTorch's own fp8 cast"
  "cross_validation_gptq:GPTQ INT4 + INT8, group-wise, g_idx, BF16 + F32:GPTQModel dequantize_weight"
  "cross_validation_awq:AWQ INT4, per-group, BF16 + F32:AutoAWQ unpack_awq + reverse_awq_order"
  "cross_validation_bnb:BitsAndBytes NF4 / FP4 / INT8, BF16 + F32:bitsandbytes dequantize_4bit"
  "cross_validation_bnb_encode:BitsAndBytes NF4 encode (byte-exact on disk):bitsandbytes on-disk bytes"
  "cross_validation_gguf:all 22 GGUF block-quant kernels, BF16 + F32:gguf-py (mirrors ggml-quants.c)"
  "cross_validation_ollama:GGUF Q8_0 from a real Ollama blob:gguf-py"
  "cross_validation_npz:NPZ / NPY parsing:NumPy"
  "cross_validation_pth:PyTorch .pth (pickle + tensor recovery):torch.load"
  "cross_validation_convert:format-conversion pipeline:per-format references"
  "cross_validation:end-to-end dequantisation:per-format references"
)

FAILED=0
for entry in "${SUITES[@]}"; do
  target="${entry%%:*}"
  rest="${entry#*:}"
  what="${rest%%:*}"
  oracle="${rest#*:}"
  printf '%-34s %s\n' "$target" "$what"
  printf '%-34s   reference: %s\n' "" "$oracle"
  if cargo test --release --all-features --test "$target" -- --quiet >/tmp/amn-verify.log 2>&1; then
    # Count from the summary line, not from '^test .* ok$': --quiet prints a dot
    # per test and never those lines, so the old pattern matched nothing and
    # every suite reported "PASS 0 tests". A zero count is now a FAILURE rather
    # than a silent pass, because "the binary compiled and ran no tests" and
    # "22 kernels verified" must not look identical in a script whose entire job
    # is substantiating a correctness claim.
    count=$(sed -n 's/.*test result: ok\. \([0-9]*\) passed.*/\1/p' /tmp/amn-verify.log \
            | awk '{ s += $1 } END { print s + 0 }')
    if [ "$count" -eq 0 ]; then
      printf '%-34s   \033[31mFAIL\033[0m  ran 0 tests (suite filtered out or empty)\n\n' ""
      FAILED=1
    else
      printf '%-34s   \033[32mPASS\033[0m  %s tests\n\n' "" "$count"
    fi
  else
    printf '%-34s   \033[31mFAIL\033[0m\n\n' ""
    tail -30 /tmp/amn-verify.log
    FAILED=1
  fi
done

cat <<'CAVEAT'
------------------------------------------------------------------------
What this proves, and what it does not

PROVES: anamnesis's dequantisation output matches goldens committed to this
repository, byte for byte. Since v0.7.4 that includes a full-width F32
comparison with no tolerance at all for *every* dequantising family: all 22
GGUF kernels, plus FP8, GPTQ, AWQ and BitsAndBytes. That matters because a
BF16 comparison discards 16 mantissa bits, and it is what caught a 1-ULP
defect in the BnB INT8 kernel that five releases of BF16 testing had passed.

DOES NOT PROVE: that those goldens are themselves correct. They were
generated by each format's own canonical library (see the "reference" line
per suite), but this run does not re-derive them.

To re-derive the goldens from the references, which is the stronger check:

  pip install gguf numpy
  python tests/fixtures/gguf_reference/generate_gguf.py --upgrade

That rebuilds every GGUF fixture from its own raw quantised bytes, needs no
model download, and refuses to proceed unless the regenerated BF16 golden
reproduces the committed one byte for byte. The other families' generators
need their respective libraries (torch, bitsandbytes, autoawq, gptqmodel)
and the source models; see tests/fixtures/*/ for each generator.

Fixture provenance is a first-class concern here: hand-rolled reference
reimplementations are banned from the generators, because three bugs once
shipped green behind circular fixtures (v0.6.4).
------------------------------------------------------------------------
CAVEAT

exit "$FAILED"