#!/usr/bin/env bash
# crawl_verify.sh — ADR-005 Phase 1b Design C correctness fixture builder.
# Runs llama-cli and hf2q on the same GGUF + prompt at T=0 greedy, compares
# their outputs by longest common byte prefix, classifies divergence, and
# (with --commit) writes tests/fixtures/{crawl_baseline.tokens,
# llama_cpp_reference.tokens, crawl_verified.meta}.
#
# Usage: scripts/crawl_verify.sh <gguf_path> [--commit] [--prompt-file PATH]

set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." && pwd)"
cd "$REPO_ROOT"

DEFAULT_PROMPT="tests/bench_prompt_128.txt"
HF2Q_BIN="target/release/hf2q"
OUT_HF2Q="/tmp/crawl_hf2q.txt"; LOG_HF2Q="/tmp/crawl_hf2q.log"
OUT_LLAMA="/tmp/crawl_llama.txt"; LOG_LLAMA="/tmp/crawl_llama.log"
RENDERED_PROMPT="/tmp/crawl_rendered_prompt.txt"; LOG_RENDER="/tmp/crawl_render.log"
# 1bNEW.19: BOS-stripped copy of the rendered prompt for llama-completion. See
# the long comment block at the llama-completion invocation for the why.
RENDERED_PROMPT_LLAMA="/tmp/crawl_rendered_prompt_nobos.txt"

usage() {
  cat <<EOF
Usage: scripts/crawl_verify.sh <gguf_path> [--commit] [--prompt-file PATH]
  <gguf_path>         Path to the Gemma 4 GGUF model (required)
  --commit            Write fixtures to tests/fixtures/ on success
  --prompt-file PATH  Override the canonical prompt (default: $DEFAULT_PROMPT)
EOF
}
err() { echo "error: $*" >&2; exit 1; }

COMMIT=0; PROMPT_FILE="$DEFAULT_PROMPT"; GGUF_PATH=""
while [[ $# -gt 0 ]]; do
  case "$1" in
    -h|--help) usage; exit 0 ;;
    --commit) COMMIT=1; shift ;;
    --prompt-file)
      [[ $# -ge 2 ]] || err "--prompt-file requires an argument"
      PROMPT_FILE="$2"; shift 2 ;;
    -*) usage >&2; err "unknown flag: $1" ;;
    *)  [[ -z "$GGUF_PATH" ]] || err "unexpected positional arg: $1"
        GGUF_PATH="$1"; shift ;;
  esac
done
[[ -n "$GGUF_PATH" ]] || { usage >&2; exit 1; }

# Validate inputs and locate tools ------------------------------------------
[[ -f "$GGUF_PATH"   ]] || err "GGUF file not found: $GGUF_PATH"
[[ -f "$PROMPT_FILE" ]] || err "Prompt file not found: $PROMPT_FILE"
[[ -x "$HF2Q_BIN"    ]] || err "hf2q binary not found at $HF2Q_BIN (run: cargo build --release)"

# llama-completion is the headless single-shot binary; llama-cli is the chat
# REPL and will hang waiting for stdin even with prompt-from-file. We MUST use
# llama-completion to get a deterministic 128-token output and then exit.
if command -v llama-completion >/dev/null 2>&1; then
  LLAMA_BIN="$(command -v llama-completion)"
elif [[ -x "/opt/llama.cpp/build/bin/llama-completion" ]]; then
  LLAMA_BIN="/opt/llama.cpp/build/bin/llama-completion"
else
  err "llama-completion not found on PATH or at /opt/llama.cpp/build/bin/llama-completion"
fi

if   command -v shasum    >/dev/null 2>&1; then SHA256_CMD=(shasum -a 256)
elif command -v sha256sum >/dev/null 2>&1; then SHA256_CMD=(sha256sum)
else err "Neither shasum nor sha256sum is available"
fi
GGUF_SHA256="$("${SHA256_CMD[@]}" "$GGUF_PATH" | awk '{print $1}')"
GIT_HEAD="$(git rev-parse HEAD 2>/dev/null || echo unknown)"

echo "=== Crawl Verify — ADR-005 Phase 1b Design C ==="
echo "GGUF:        $GGUF_PATH"
echo "GGUF sha256: $GGUF_SHA256"
echo "Prompt:      $PROMPT_FILE"
echo "hf2q:        $HF2Q_BIN"
echo "llama-comp:  $LLAMA_BIN"
echo "git HEAD:    $GIT_HEAD"
echo
echo "--- Chat template probe (best-effort) ---"
if command -v strings >/dev/null 2>&1; then
  # `set -o pipefail` + `grep -m1` produces SIGPIPE on `strings` once grep exits,
  # which trips the `||` even when grep matched. Capture into a variable first.
  CHAT_PROBE="$(strings "$GGUF_PATH" 2>/dev/null | grep -m1 -i "chat_template" || true)"
  if [[ -n "$CHAT_PROBE" ]]; then
    echo "$CHAT_PROBE"
  else
    echo "(no tokenizer.chat_template string found via 'strings')"
  fi
else
  echo "(strings not available; skipping template probe)"
fi
echo

# Pre-render the chat template via hf2q -------------------------------------
# ADR-005 1bNEW.0c: the previous version of this script passed `--jinja` to
# llama-completion, which routes through a different prompt path than the one
# hf2q uses — producing thought-channel output (`<|channel>thought\n\n*`)
# regardless of hf2q's actual correctness state. The byte-prefix classification
# was structurally stuck at RED (byte 0) (see ADR line 198).
#
# Fix: hf2q has HF2Q_DUMP_RENDERED_PROMPT=<path> which writes its fully
# chat-templated prompt to the given file and exits. We then feed the
# rendered text to llama-completion WITHOUT `--jinja`, so both tools see
# the byte-identical rendered prompt.
echo "--- Pre-rendering chat template via hf2q ($RENDERED_PROMPT) ---"
if ! HF2Q_DUMP_RENDERED_PROMPT="$RENDERED_PROMPT" \
      "$HF2Q_BIN" generate --model "$GGUF_PATH" --prompt-file "$PROMPT_FILE" \
        --max-tokens 1 --temperature 0 \
        >/dev/null 2>"$LOG_RENDER"; then
  echo "hf2q render-only failed. See $LOG_RENDER" >&2; exit 3
fi
[[ -s "$RENDERED_PROMPT" ]] || err "hf2q render produced empty $RENDERED_PROMPT (see $LOG_RENDER)"
echo "hf2q rendered prompt: $(wc -c < "$RENDERED_PROMPT" | tr -d ' ') bytes"

# 1bNEW.19: strip the leading literal `<bos>` text for the llama-completion
# input. Rationale (full Chesterton-fence trace):
#
#   * hf2q's rendered prompt begins with the 5 literal bytes `<bos>` followed
#     by `<|turn>user\n...`. hf2q's tokenizer parses `<bos>` as the special
#     BOS token (id 2), giving a 187-token sequence `[2, 105, 2364, ...]`.
#   * 1bNEW.0c switched to feeding the same rendered file to llama-completion
#     via `--file` (so both tools see byte-identical input). It correctly
#     dropped `--jinja` and added `-no-cnv` so llama-completion would NOT
#     re-apply the chat template.
#   * What 0c missed: llama-completion's prompt path still calls
#     `common_tokenize(ctx, prompt, /*add_special=*/true, /*parse_special=*/true)`
#     at /opt/llama.cpp/tools/completion/completion.cpp:322. With
#     `add_special=true` and `parse_special=true`:
#       - `parse_special=true` makes the literal `<bos>` text resolve to BOS
#         token id 2 (the same way hf2q does it).
#       - `add_special=true` AND vocab `add_bos=true` makes
#         /opt/llama.cpp/src/llama-vocab.cpp:3081 push another BOS token at
#         position 0 BEFORE the parsed-special BOS.
#     Result: llama.cpp's token sequence is `[2, 2, 105, 2364, ...]` (188
#     tokens, two BOS) while hf2q's is `[2, 105, 2364, ...]` (187 tokens,
#     one BOS). Every position index is shifted by one, every per-position
#     hidden state is non-comparable, and the byte-prefix classification is
#     stuck at the BOS-shift artifact.
#   * llama-vocab.cpp itself prints
#       check_double_bos_eos: Added a BOS token to the prompt as specified
#       by the model but the prompt also starts with a BOS token. So now
#       the final prompt starts with 2 BOS tokens.
#     when it hits this case (llama-vocab.cpp:3111-3115).
#   * Why we don't use `--override-kv tokenizer.ggml.add_bos_token=bool:false`:
#     Gemma 4 has a hardcoded workaround at
#     /opt/llama.cpp/src/llama-vocab.cpp:2329-2335 that re-forces
#     `add_bos = true` for `LLAMA_VOCAB_PRE_TYPE_GEMMA4` regardless of
#     metadata, so the override is silently ignored.
#   * Why we don't use `--prompt` instead of `--file`: same call site, same
#     `common_tokenize(..., true, true)` invocation, same auto-BOS.
#   * Why we don't use llama-server: heavier dependency change, separate
#     prompt path, defeats the "byte-identical input" guarantee 0c
#     established.
#
# Fix: rewrite the rendered prompt with the leading `<bos>` (5 bytes) stripped
# off. llama.cpp then auto-prepends exactly one BOS via add_special=true and
# we get the same `[2, 105, 2364, ...]` 187-token sequence as hf2q.
#
# Citations:
#   - Spike C report: docs/spike-C-results.md:173-218 ("Prompt alignment
#     (CRITICAL — pre-existing crawl_verify.sh hazard)")
#   - llama-completion tokenize call: /opt/llama.cpp/tools/completion/completion.cpp:322
#   - SPM auto-BOS gating logic: /opt/llama.cpp/src/llama-vocab.cpp:3081
#   - SPM double-BOS warning: /opt/llama.cpp/src/llama-vocab.cpp:3111-3115
#   - Gemma 4 forced add_bos workaround: /opt/llama.cpp/src/llama-vocab.cpp:2329-2335
echo "--- Stripping leading <bos> text for llama-completion ($RENDERED_PROMPT_LLAMA) ---"
python3 - "$RENDERED_PROMPT" "$RENDERED_PROMPT_LLAMA" <<'PY'
import sys
src, dst = sys.argv[1], sys.argv[2]
data = open(src, "rb").read()
if not data.startswith(b"<bos>"):
    sys.stderr.write(
        "error: rendered prompt does not start with literal '<bos>'; "
        "hf2q chat template changed and the 1bNEW.19 BOS-strip assumption "
        "is no longer valid. Re-investigate before proceeding.\n"
    )
    sys.exit(1)
open(dst, "wb").write(data[5:])
PY
[[ -s "$RENDERED_PROMPT_LLAMA" ]] || err "BOS-stripped prompt is empty"
echo "BOS-stripped prompt: $(wc -c < "$RENDERED_PROMPT_LLAMA" | tr -d ' ') bytes (was $(wc -c < "$RENDERED_PROMPT" | tr -d ' '))"

# Run llama-completion and hf2q ----------------------------------------------
# Notes:
#   -st / --single-turn   : exit after one turn (otherwise waits for stdin)
#   </dev/null            : explicit EOF on stdin so no interactive prompt
#   --no-display-prompt   : stdout = generated text only, no echo
#   -no-cnv               : disable conversation mode (auto-enabled when the
#                           GGUF has a chat template). Without this, llama-
#                           completion tries to re-apply the template to our
#                           already-rendered input and aborts with
#                           "this custom template is not supported".
#   NO --jinja            : we pass the already-rendered prompt as a raw file,
#                           so llama-completion must NOT re-apply the template.
#   --file <stripped>     : 1bNEW.19 — feed the BOS-stripped copy so
#                           llama-completion's auto-BOS produces exactly one
#                           BOS at position 0, matching hf2q's 187-token
#                           sequence. See long comment above.
echo "--- Running llama-completion (T=0 greedy, 128 tokens, BOS-stripped prompt) ---"
if ! "$LLAMA_BIN" --model "$GGUF_PATH" --file "$RENDERED_PROMPT_LLAMA" \
      --predict 128 --temp 0 --seed 42 \
      --no-display-prompt -no-cnv \
      -st -ngl 999 \
      </dev/null >"$OUT_LLAMA" 2>"$LOG_LLAMA"; then
  echo "llama-completion failed. See $LOG_LLAMA" >&2; exit 3
fi

# 1bNEW.19 sanity gate: llama.cpp must report exactly the same prompt token
# count as hf2q (187 on the canonical bench prompt). If we ever see a
# mismatch here, the comparison is structurally invalid and the byte-prefix
# classification is meaningless — fail loudly instead of silently producing
# a wrong RED/YELLOW/GREEN reading.
LLAMA_PROMPT_TOKENS="$(grep -oE 'prompt eval time =[^/]*/[[:space:]]*[0-9]+ tokens' "$LOG_LLAMA" | grep -oE '[0-9]+ tokens' | head -1 | awk '{print $1}')"
if [[ -z "$LLAMA_PROMPT_TOKENS" ]]; then
  echo "warn: could not parse llama-completion prompt token count from $LOG_LLAMA" >&2
else
  echo "llama-completion prompt tokens: $LLAMA_PROMPT_TOKENS"
fi
if grep -q "prompt also starts with a BOS token" "$LOG_LLAMA"; then
  echo "error: llama-completion still reports double-BOS — 1bNEW.19 fix is broken." >&2
  echo "       See $LOG_LLAMA for the check_double_bos_eos warning." >&2
  exit 4
fi

echo "--- Running hf2q (T=0 greedy, 128 tokens) ---"
if ! "$HF2Q_BIN" generate --model "$GGUF_PATH" --prompt-file "$PROMPT_FILE" \
      --max-tokens 128 --temperature 0 \
      >"$OUT_HF2Q" 2>"$LOG_HF2Q"; then
  echo "hf2q failed. See $LOG_HF2Q" >&2; exit 3
fi

# Compare via portable python one-shot --------------------------------------
command -v python3 >/dev/null 2>&1 || err "python3 is required for byte-prefix comparison"
read -r LLAMA_BYTES HF2Q_BYTES COMMON_BYTES DIVERGE_A DIVERGE_B < <(
  python3 - "$OUT_LLAMA" "$OUT_HF2Q" <<'PY'
import sys, json
a = open(sys.argv[1], "rb").read()
b = open(sys.argv[2], "rb").read()
n = min(len(a), len(b)); i = 0
while i < n and a[i] == b[i]: i += 1
def snip(buf, start, length=200):
    s = buf[start:start+length].decode("utf-8", errors="replace")
    if len(buf) - start > length: s += "..."
    return json.dumps(s)
print(len(a), len(b), i, snip(a, i), snip(b, i))
PY
)
APPROX_TOKENS=$(( COMMON_BYTES / 4 ))

echo
echo "--- Comparison ---"
echo "llama-cli output: ${LLAMA_BYTES} bytes"
echo "hf2q output:      ${HF2Q_BYTES} bytes"
echo "Common prefix:    ${COMMON_BYTES} bytes (~${APPROX_TOKENS} tokens, ~4 chars/tok)"

# Classify -------------------------------------------------------------------
if   [[ "$COMMON_BYTES" -eq "$LLAMA_BYTES" && "$COMMON_BYTES" -eq "$HF2Q_BYTES" ]]; then
  CLASS="PERFECT"; CLASS_MSG="Token-for-token match with llama.cpp."
elif [[ "$COMMON_BYTES" -lt 20  ]]; then
  CLASS="RED";     CLASS_MSG="Chat template mismatch or major math bug. hf2q is NOT porting llama.cpp's pipeline correctly."
elif [[ "$COMMON_BYTES" -lt 200 ]]; then
  CLASS="YELLOW";  CLASS_MSG="Early FP drift. Partial port correctness."
else
  CLASS="GREEN";   CLASS_MSG="Normal FP-associativity drift; Crawl is real."
fi
echo
echo "Classification: $CLASS"
echo "  $CLASS_MSG"
echo
echo "--- Divergence point (first 200 chars after byte $COMMON_BYTES) ---"
echo "llama-cli: $DIVERGE_A"
echo "hf2q:      $DIVERGE_B"
echo

# Optionally commit fixtures -------------------------------------------------
if [[ "$COMMIT" -eq 1 ]]; then
  FIX_DIR="tests/fixtures"; mkdir -p "$FIX_DIR"
  cp "$OUT_HF2Q"  "$FIX_DIR/crawl_baseline.tokens"
  cp "$OUT_LLAMA" "$FIX_DIR/llama_cpp_reference.tokens"
  CLEAN=$(git diff --quiet && git diff --cached --quiet && echo yes || echo no)
  {
    echo "# ADR-005 Phase 1b Design C — crawl verification metadata"
    echo "timestamp_utc:      $(date -u +%Y-%m-%dT%H:%M:%SZ)"
    echo "git_head:           $GIT_HEAD"
    echo "git_status_clean:   $CLEAN"
    echo "gguf_path:          $GGUF_PATH"
    echo "gguf_sha256:        $GGUF_SHA256"
    echo "prompt_file:        $PROMPT_FILE"
    echo "llama_cli:          $LLAMA_BIN"
    echo "llama_bytes:        $LLAMA_BYTES"
    echo "hf2q_bytes:         $HF2Q_BYTES"
    echo "common_prefix:      $COMMON_BYTES"
    echo "approx_tokens:      $APPROX_TOKENS"
    echo "classification:     $CLASS"
    echo "classification_msg: $CLASS_MSG"
  } > "$FIX_DIR/crawl_verified.meta"
  echo "Fixtures committed to $FIX_DIR/. Run 'git add $FIX_DIR && git commit' to persist."
fi

[[ "$CLASS" == "RED" ]] && exit 2 || exit 0