kibble 0.1.0

//! Deterministic spherical k-means topic clustering of dataset answers.

use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::path::Path;

const MAX_ITERS: usize = 50;

fn splitmix64(state: &mut u64) -> u64 {
    *state = state.wrapping_add(0x9E3779B97F4A7C15);
    let mut z = *state;
    z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9);
    z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB);
    z ^ (z >> 31)
}

fn normalize(v: &mut [f32]) {
    let n: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
    if n > 0.0 {
        for x in v.iter_mut() {
            *x /= n;
        }
    }
}

fn dot(a: &[f32], b: &[f32]) -> f32 {
    a.iter().zip(b).map(|(x, y)| x * y).sum()
}

/// Spherical k-means over unit-normalized vectors. Deterministic (fixed seed,
/// `pts[0]` as the first centroid). Returns (assignment per input, centroids).
pub fn kmeans(vecs: &[Vec<f32>], k: usize) -> (Vec<usize>, Vec<Vec<f32>>) {
    let n = vecs.len();
    if n == 0 {
        return (Vec::new(), Vec::new());
    }
    let k = k.clamp(1, n);
    let dim = vecs[0].len();

    let mut pts: Vec<Vec<f32>> = vecs.to_vec();
    for p in pts.iter_mut() {
        normalize(p);
    }

    // k-means++ seeding with a deterministic PRNG.
    let mut seed = 0x00C0_FFEE_u64;
    let mut centroids: Vec<Vec<f32>> = vec![pts[0].clone()];
    while centroids.len() < k {
        let d2: Vec<f32> = pts.iter().map(|p| {
            let best = centroids.iter().map(|c| dot(p, c)).fold(f32::MIN, f32::max);
            (1.0 - best).max(0.0)
        }).collect();
        let sum: f32 = d2.iter().sum();
        let r = (splitmix64(&mut seed) as f64 / u64::MAX as f64) as f32 * sum;
        let mut acc = 0.0;
        let mut chosen = d2.len() - 1;
        for (i, &d) in d2.iter().enumerate() {
            acc += d;
            if acc >= r {
                chosen = i;
                break;
            }
        }
        centroids.push(pts[chosen].clone());
    }

    let mut assign = vec![0usize; n];
    for _ in 0..MAX_ITERS {
        let mut changed = false;
        for (i, p) in pts.iter().enumerate() {
            let mut best = 0;
            let mut bestsim = f32::MIN;
            for (c, cen) in centroids.iter().enumerate() {
                let s = dot(p, cen);
                if s > bestsim {
                    bestsim = s;
                    best = c;
                }
            }
            if assign[i] != best {
                assign[i] = best;
                changed = true;
            }
        }
        let mut sums = vec![vec![0f32; dim]; k];
        let mut counts = vec![0usize; k];
        for (i, p) in pts.iter().enumerate() {
            let c = assign[i];
            for d in 0..dim {
                sums[c][d] += p[d];
            }
            counts[c] += 1;
        }
        for c in 0..k {
            if counts[c] > 0 {
                for s in &mut sums[c] {
                    *s /= counts[c] as f32;
                }
                normalize(&mut sums[c]);
                centroids[c] = std::mem::take(&mut sums[c]);
            }
        }
        if !changed {
            break;
        }
    }
    (assign, centroids)
}

/// Common English stopwords filtered out of cluster labels so distinctive terms
/// win instead of grammar particles (e.g. `zebra-savanna`, not `the-zebra-to`).
const STOPWORDS: &[&str] = &[
    "the", "a", "an", "and", "or", "but", "to", "of", "in", "on", "at", "by",
    "for", "with", "as", "is", "are", "was", "were", "be", "been", "it", "its",
    "this", "that", "these", "those", "from", "into", "if", "then", "than", "so",
];

fn is_stopword(term: &str) -> bool {
    STOPWORDS.contains(&term)
}

/// A token worth using as a topic-label term: at least 3 chars, not a pure number, not a
/// hex/address literal, and not majority-digits. Filters the tokenization noise (`0`, `i`, `td`,
/// `0x0008`, `2024`) that otherwise dominates TF-IDF and produces junk labels, while keeping short
/// tech terms (tcp, bgp, i2p, gan).
fn is_meaningful_term(tok: &str) -> bool {
    let n = tok.chars().count();
    if n < 3 {
        return false;
    }
    if tok.starts_with("0x") {
        return false;
    }
    let digits = tok.chars().filter(|c| c.is_ascii_digit()).count();
    if digits == n {
        return false; // pure number
    }
    digits * 2 < n // reject majority-digit tokens
}

/// Label each cluster by its top distinctive terms (frequent in-cluster, rare
/// across clusters). Stopwords and tokenization noise are filtered so grammar particles
/// and digit/hex fragments never dominate. Deterministic. Empty cluster → `topic-<c>`.
pub fn label_clusters(texts: &[String], assignment: &[usize], k: usize) -> Vec<String> {
    let mut cluster_tf: Vec<HashMap<String, usize>> = vec![HashMap::new(); k];
    let mut term_clusters: HashMap<String, HashSet<usize>> = HashMap::new();
    for (i, t) in texts.iter().enumerate() {
        let c = assignment[i];
        if c >= k {
            continue;
        }
        for tok in crate::bm25::tokenize(t) {
            if is_stopword(&tok) || !is_meaningful_term(&tok) {
                continue;
            }
            *cluster_tf[c].entry(tok.clone()).or_insert(0) += 1;
            term_clusters.entry(tok).or_default().insert(c);
        }
    }
    (0..k).map(|c| {
        let mut scored: Vec<(String, f64)> = cluster_tf[c].iter().map(|(term, &tf)| {
            let cf = term_clusters.get(term).map(|s| s.len()).unwrap_or(1) as f64;
            let idf = (k as f64 / cf).ln() + 1.0;
            (term.clone(), tf as f64 * idf)
        }).collect();
        scored.sort_by(|a, b| {
            b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal).then(a.0.cmp(&b.0))
        });
        let top: Vec<String> = scored.into_iter().take(3).map(|(t, _)| t).collect();
        if top.is_empty() { format!("topic-{c}") } else { top.join("-") }
    }).collect()
}

/// Build the LLM prompt that names all topics at once from their distinctive
/// terms + a sample. Requests a JSON array of names in topic order.
fn build_label_prompt(labels: &[String], samples: &[String]) -> String {
    let k = labels.len();
    let mut p = format!(
        "You will name exactly {k} dataset topics. Return ONLY a JSON array of exactly {k} short \
         titles (2-4 words, Title Case), one per topic, in the SAME order as listed, and nothing \
         else. The array MUST have exactly {k} elements.\n\n",
    );
    for (i, (l, s)) in labels.iter().zip(samples.iter()).enumerate() {
        let terms = l.replace('-', ", ");
        let sample: String = s.chars().take(300).collect::<String>().replace('\n', " ");
        p.push_str(&format!("Topic {}: key terms: {}; sample: {}\n", i + 1, terms, sample));
    }
    p
}

/// Parse the LLM's reply into exactly `k` non-empty topic names (≤48 chars each).
/// Extracts the `[...]` array (tolerating code fences / surrounding prose). `None`
/// if the array is absent/unparseable, the count differs, or any name is empty.
fn parse_labels(response: &str, k: usize) -> Option<Vec<String>> {
    let start = response.find('[')?;
    let end = response.rfind(']')?;
    if end < start {
        return None;
    }
    let arr: Vec<String> = serde_json::from_str(&response[start..=end]).ok()?;
    if arr.len() != k {
        return None;
    }
    let cleaned: Vec<String> = arr.iter().map(|s| s.trim().chars().take(48).collect::<String>()).collect();
    if cleaned.iter().any(|s| s.is_empty()) {
        return None;
    }
    Some(cleaned)
}

/// Opt-in post-pass: replace `result.labels` with LLM-generated topic names via the
/// `[ask]` endpoint. No-op unless `[cluster].llm_labels` and `[ask].base_url` are set.
/// Fail-soft: any failure keeps the existing (term-based) labels; never errors.
pub async fn apply_llm_labels(cfg: &crate::config::KibbleConfig, result: &mut ClusterResult) {
    if !cfg.cluster.llm_labels || cfg.ask.base_url.is_empty() || result.labels.is_empty() {
        return;
    }
    let prompt = build_label_prompt(&result.labels, &result.samples);
    let messages = vec![
        serde_json::json!({"role": "system", "content": "You name dataset topics concisely and always return a JSON array with exactly the requested number of items."}),
        serde_json::json!({"role": "user", "content": prompt}),
    ];
    let client = match crate::net::build_client(cfg.network.proxy.as_deref()) {
        Ok(c) => c,
        Err(e) => { eprintln!("kibble: LLM labels skipped ({e}) — keeping term labels"); return; }
    };
    let key = crate::llm::env_key(&["ASK_API_KEY", "OPENAI_API_KEY"]);
    match crate::llm::chat_turn(&client, &cfg.ask.base_url, &cfg.ask.model, &key, &messages, None, crate::llm::Sampling::recommended(0.0, cfg.ask.max_tokens)).await {
        Ok((Some(content), ..)) => match parse_labels(&content, result.labels.len()) {
            Some(names) => result.labels = names,
            None => eprintln!("kibble: LLM labels skipped (unparseable response) — keeping term labels"),
        },
        Ok((None, ..)) => {}
        Err(e) => eprintln!("kibble: LLM labels skipped ({e}) — keeping term labels"),
    }
}

/// Separator between hierarchy levels in an auto-topic path (U+203A with spaces).
pub const HIERARCHICAL_SEPARATOR: &str = " › ";

fn build_hier_label_prompt(labels: &[String], samples: &[String]) -> String {
    let k = labels.len();
    let mut p = format!(
        "You will name exactly {k} dataset topics, each as a hierarchical category path from broad \
         to specific using '{sep}' between levels (2-3 levels, Title Case), e.g. \
         \"Systems{sep}Linux{sep}Memory\". Return ONLY a JSON array of exactly {k} path strings, one \
         per topic, in the SAME order as listed, and nothing else. The array MUST have exactly {k} \
         elements.\n\n",
        sep = HIERARCHICAL_SEPARATOR,
    );
    for (i, (l, s)) in labels.iter().zip(samples.iter()).enumerate() {
        let terms = l.replace('-', ", ");
        let sample: String = s.chars().take(300).collect::<String>().replace('\n', " ");
        p.push_str(&format!("Topic {}: key terms: {}; sample: {}\n", i + 1, terms, sample));
    }
    p
}

/// Ask the [ask] LLM for hierarchical category paths (one per cluster), reusing the same
/// chat + parse path as `apply_llm_labels`. Returns `None` (keep prior labels) on any failure:
/// no [ask] endpoint, transport error, or unparseable/wrong-count response.
pub async fn hier_labels(cfg: &crate::config::KibbleConfig, term_labels: &[String], samples: &[String]) -> Option<Vec<String>> {
    if cfg.ask.base_url.is_empty() || term_labels.is_empty() {
        return None;
    }
    let prompt = build_hier_label_prompt(term_labels, samples);
    let messages = vec![
        serde_json::json!({"role": "system", "content": "You name dataset topics as concise hierarchical paths and always return a JSON array with exactly the requested number of items."}),
        serde_json::json!({"role": "user", "content": prompt}),
    ];
    let client = crate::net::build_client(cfg.network.proxy.as_deref()).ok()?;
    let key = crate::llm::env_key(&["ASK_API_KEY", "OPENAI_API_KEY"]);
    match crate::llm::chat_turn(&client, &cfg.ask.base_url, &cfg.ask.model, &key, &messages, None, crate::llm::Sampling::recommended(0.0, cfg.ask.max_tokens)).await {
        Ok((Some(content), ..)) => parse_labels(&content, term_labels.len()),
        _ => None,
    }
}

/// The clustering artifact written to `clusters.json`.
#[derive(Serialize, Deserialize, Clone)]
pub struct ClusterResult {
    pub k: usize,
    pub model: String,
    pub centroids: Vec<Vec<f32>>,
    pub labels: Vec<String>,
    pub sizes: Vec<usize>,
    pub samples: Vec<String>,
}

/// Merge clusters smaller than `min_size` into their nearest surviving centroid.
/// `min_size <= 1` → unchanged. Unchanged also when no cluster reaches `min_size`
/// or when every cluster already does. Deterministic: nearest by cosine (lowest-index
/// tie-break via `assign_topics`), survivors recomputed as renormalized means, and
/// clusters reindexed to `0..survivors`.
fn merge_small_clusters(assign: &[usize], centroids: &[Vec<f32>], vecs: &[Vec<f32>], min_size: usize) -> (Vec<usize>, Vec<Vec<f32>>) {
    if min_size <= 1 {
        return (assign.to_vec(), centroids.to_vec());
    }
    let k = centroids.len();
    let mut counts = vec![0usize; k];
    for &c in assign {
        if c < k { counts[c] += 1; }
    }
    let survivors: Vec<usize> = (0..k).filter(|&c| counts[c] >= min_size).collect();
    if survivors.is_empty() || survivors.len() == k {
        return (assign.to_vec(), centroids.to_vec());
    }
    let surviving_centroids: Vec<Vec<f32>> = survivors.iter().map(|&c| centroids[c].clone()).collect();
    let new_assign = assign_topics(&surviving_centroids, vecs);
    let dim = surviving_centroids[0].len();
    let mut sums = vec![vec![0f32; dim]; survivors.len()];
    let mut newcounts = vec![0usize; survivors.len()];
    for (i, &c) in new_assign.iter().enumerate() {
        for d in 0..dim { sums[c][d] += vecs[i][d]; }
        newcounts[c] += 1;
    }
    let mut out_centroids = Vec::with_capacity(survivors.len());
    for c in 0..survivors.len() {
        if newcounts[c] > 0 {
            for s in &mut sums[c] { *s /= newcounts[c] as f32; }
            normalize(&mut sums[c]);
            out_centroids.push(std::mem::take(&mut sums[c]));
        } else {
            // A survivor keeps all its own members after reassignment (dropping non-surviving
            // rivals can only add members, never remove them), so this is unreachable when k-means
            // converged. It guards the rare stale-assignment case (k-means hit MAX_ITERS): keep the
            // pre-merge centroid rather than emitting a zero vector.
            out_centroids.push(surviving_centroids[c].clone());
        }
    }
    (new_assign, out_centroids)
}

/// Pure clustering core: k-means + labels + sizes/samples. `k` is the requested
/// count; the result's `k` is the actual cluster count (clamped by `kmeans`,
/// then by `merge_small_clusters`).
pub fn cluster_from(texts: &[String], vecs: &[Vec<f32>], k: usize, model: &str, min_cluster_size: usize) -> ClusterResult {
    let (assign, centroids) = kmeans(vecs, k);
    let (assign, centroids) = merge_small_clusters(&assign, &centroids, vecs, min_cluster_size);
    let kk = centroids.len();
    let labels = label_clusters(texts, &assign, kk);
    let mut sizes = vec![0usize; kk];
    let mut first: Vec<Option<usize>> = vec![None; kk];
    for (i, &c) in assign.iter().enumerate() {
        sizes[c] += 1;
        if first[c].is_none() {
            first[c] = Some(i);
        }
    }
    let samples: Vec<String> = (0..kk).map(|c| {
        first[c].map(|i| texts[i].chars().take(80).collect::<String>()).unwrap_or_default()
    }).collect();
    ClusterResult { k: kk, model: model.to_string(), centroids, labels, sizes, samples }
}

/// Assign each vector to the centroid with the highest cosine similarity.
/// Ties break to the lowest centroid index (deterministic). One index per vector.
/// Precondition: `centroids` is non-empty (callers guard this before calling).
pub fn assign_topics(centroids: &[Vec<f32>], vecs: &[Vec<f32>]) -> Vec<usize> {
    vecs.iter().map(|v| {
        let mut best = 0usize;
        let mut best_sim = f32::MIN;
        for (i, c) in centroids.iter().enumerate() {
            let s = crate::vectors::cosine(v, c);
            if s > best_sim {
                best_sim = s;
                best = i;
            }
        }
        best
    }).collect()
}

/// Load `clusters.json`, guarding that its embedding model matches the configured
/// one (centroids are only comparable in the same space). Returns `None` on
/// missing/unparseable file, model mismatch, or empty centroids.
pub fn load_clusters(path: &Path, want_model: &str) -> Option<ClusterResult> {
    let text = std::fs::read_to_string(path).ok()?;
    let cr: ClusterResult = serde_json::from_str(&text).ok()?;
    if cr.model != want_model {
        eprintln!(
            "kibble: clusters.json model '{}' != embed model '{}' — skipping (rebuild/re-cluster)",
            cr.model, want_model
        );
        return None;
    }
    if cr.centroids.is_empty() {
        return None;
    }
    Some(cr)
}

/// Write a clustering result to `repo_root/out` as JSON, creating parent dirs.
pub fn write_clusters(repo_root: &Path, out: &str, result: &ClusterResult) -> std::io::Result<()> {
    let path = repo_root.join(out);
    if let Some(parent) = path.parent() {
        std::fs::create_dir_all(parent)?;
    }
    std::fs::write(&path, serde_json::to_string(result)?)?;
    Ok(())
}

/// Cluster the built dataset's answers and write `[cluster].out`. Fail-soft:
/// no dataset / no embed backend / embed error → `Ok(None)` with a warning.
pub async fn run_cluster(repo_root: &Path, k_override: Option<usize>) -> std::io::Result<Option<ClusterResult>> {
    let cfg = crate::config::load_config(&repo_root.join(crate::config::CONFIG_FILE));
    let k = k_override.unwrap_or(cfg.cluster.k);

    let ds = repo_root.join(&cfg.paths.dataset_dir);
    let mut answers: Vec<String> = Vec::new();
    for split in ["train.jsonl", "valid.jsonl", "test.jsonl"] {
        let p = ds.join(split);
        if p.is_file() {
            for r in crate::eval::parse_split(&p).rows {
                answers.push(r.assistant);
            }
        }
    }
    if answers.is_empty() {
        eprintln!("kibble: nothing to cluster — run `kibble build` first");
        return Ok(None);
    }

    let embed = &cfg.understand.embed;
    if embed.base_url.is_empty() {
        eprintln!("kibble: no embed backend — set [understand.embed].base_url to cluster");
        return Ok(None);
    }
    let store = repo_root.join(&embed.store);
    let vecs = match crate::embed::EndpointEmbedder::new(embed, cfg.network.proxy.as_deref()) {
        Ok(e) => match crate::vectors::get_or_embed(&e, &store, &embed.model, &answers, embed.batch_size).await {
            Ok(v) => v,
            Err(err) => {
                eprintln!("kibble: clustering skipped (embed failed): {err}");
                return Ok(None);
            }
        },
        Err(err) => {
            eprintln!("kibble: clustering skipped (embed init failed): {err}");
            return Ok(None);
        }
    };

    let mut result = cluster_from(&answers, &vecs, k, &embed.model, cfg.cluster.min_cluster_size);
    apply_llm_labels(&cfg, &mut result).await;
    write_clusters(repo_root, &cfg.cluster.out, &result)?;
    Ok(Some(result))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn run_cluster_none_paths() {
        // (a) no dataset at all → Ok(None)
        let dir = std::env::temp_dir().join(format!("kibble_clus_none_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        std::fs::write(dir.join(crate::config::CONFIG_FILE),
            "[paths]\ndataset_dir=\"data/ds\"\n[understand.embed]\nbase_url=\"\"\n").unwrap();
        assert!(run_cluster(&dir, None).await.unwrap().is_none());

        // (b) dataset present but no embed backend → Ok(None)
        std::fs::create_dir_all(dir.join("data/ds")).unwrap();
        std::fs::write(dir.join("data/ds/train.jsonl"),
            "{\"messages\":[{\"role\":\"user\",\"content\":\"q\"},{\"role\":\"assistant\",\"content\":\"an answer\"}]}\n").unwrap();
        assert!(run_cluster(&dir, None).await.unwrap().is_none());

        std::fs::remove_dir_all(&dir).ok();
    }

    #[test]
    fn cluster_from_builds_result() {
        let texts = vec![
            "alpha alpha one".to_string(), "alpha two".to_string(),
            "beta beta three".to_string(), "beta four".to_string(),
        ];
        let vecs = vec![
            vec![1.0, 0.0], vec![0.95, 0.05],
            vec![0.0, 1.0], vec![0.05, 0.95],
        ];
        let r = cluster_from(&texts, &vecs, 2, "m", 0);
        assert_eq!(r.k, 2);
        assert_eq!(r.model, "m");
        assert_eq!(r.centroids.len(), 2);
        assert_eq!(r.labels.len(), 2);
        assert_eq!(r.sizes.iter().sum::<usize>(), 4);
        assert_eq!(r.samples.len(), 2);
    }

    #[test]
    fn merge_small_clusters_folds_singleton() {
        // 3 vectors near [1,0] + 1 near [0,1]; kmeans(k=2) → a size-3 and a size-1 cluster.
        let vecs = vec![vec![1.0, 0.0], vec![0.95, 0.05], vec![0.9, 0.1], vec![0.0, 1.0]];
        let (assign, centroids) = kmeans(&vecs, 2);
        let (m_assign, m_cents) = merge_small_clusters(&assign, &centroids, &vecs, 2);
        assert_eq!(m_cents.len(), 1, "the singleton is merged into the surviving cluster");
        assert!(m_assign.iter().all(|&c| c == 0), "all vectors reassigned to the sole survivor");
    }

    #[test]
    fn merge_small_clusters_noop_cases() {
        let vecs = vec![vec![1.0, 0.0], vec![0.0, 1.0]];
        let (assign, cents) = kmeans(&vecs, 2);
        // min_size <= 1 → unchanged
        assert_eq!(merge_small_clusters(&assign, &cents, &vecs, 1), (assign.clone(), cents.clone()));
        // all clusters tiny (both size 1, min_size 2 → no survivors) → unchanged
        assert_eq!(merge_small_clusters(&assign, &cents, &vecs, 2), (assign, cents));
    }

    #[test]
    fn cluster_from_merges_when_min_size_set() {
        let texts: Vec<String> = (0..4).map(|i| format!("doc {i}")).collect();
        let vecs = vec![vec![1.0, 0.0], vec![0.95, 0.05], vec![0.9, 0.1], vec![0.0, 1.0]];
        let merged = cluster_from(&texts, &vecs, 2, "m", 2);
        assert_eq!(merged.k, 1, "the size-1 topic is merged away");
        let unmerged = cluster_from(&texts, &vecs, 2, "m", 0);
        assert_eq!(unmerged.k, 2, "min_size 0 → no merge (today's behavior)");
    }

    #[test]
    fn kmeans_separates_groups_deterministically() {
        // Three tight groups along the axes.
        let vecs = vec![
            vec![1.0, 0.0, 0.0], vec![0.9, 0.1, 0.0],
            vec![0.0, 1.0, 0.0], vec![0.1, 0.9, 0.0],
            vec![0.0, 0.0, 1.0], vec![0.0, 0.1, 0.9],
        ];
        let (a, cents) = kmeans(&vecs, 3);
        assert_eq!(cents.len(), 3);
        assert_eq!(a[0], a[1]); // group 1 together
        assert_eq!(a[2], a[3]); // group 2 together
        assert_eq!(a[4], a[5]); // group 3 together
        assert_ne!(a[0], a[2]);
        assert_ne!(a[2], a[4]);
        // deterministic across runs
        let (a2, _) = kmeans(&vecs, 3);
        assert_eq!(a, a2);
    }

    #[test]
    fn kmeans_clamps_and_handles_empty() {
        assert_eq!(kmeans(&[], 3), (Vec::new(), Vec::new()));
        let one = vec![vec![1.0, 0.0]];
        let (a, c) = kmeans(&one, 5); // k > n → clamp to 1
        assert_eq!(c.len(), 1);
        assert_eq!(a, vec![0]);
    }

    #[test]
    fn labels_pick_distinctive_terms() {
        // cluster 0 is about "zebra", cluster 1 about "quantum"; "the" is shared.
        let texts = vec![
            "the zebra zebra stripes".to_string(),
            "the zebra savanna".to_string(),
            "the quantum quantum physics".to_string(),
            "the quantum entanglement".to_string(),
        ];
        let assignment = vec![0, 0, 1, 1];
        let labels = label_clusters(&texts, &assignment, 2);
        assert_eq!(labels.len(), 2);
        assert!(labels[0].contains("zebra"), "got {:?}", labels[0]);
        assert!(labels[1].contains("quantum"), "got {:?}", labels[1]);
        // stopwords must never appear as a label term (each label is `-`-joined terms)
        for label in &labels {
            for term in label.split('-') {
                assert!(!is_stopword(term), "stopword {term:?} in label {label:?}");
            }
        }
    }

    #[test]
    fn labels_drop_numeric_hex_and_short_noise() {
        // cluster 0's real theme is "syscall/reserved" buried under digit/short/hex noise;
        // cluster 1's is "malloc/struct". Noise tokens must not surface as label terms.
        let texts = vec![
            "0 1 i td 0x0008 syscall syscall reserved zend".to_string(),
            "2 3 br 6 syscall reserved zend zend".to_string(),
            "s o i 99 malloc malloc struct pointer heap".to_string(),
            "1 0 42 malloc struct struct heap".to_string(),
        ];
        let assignment = vec![0, 0, 1, 1];
        let labels = label_clusters(&texts, &assignment, 2);
        for label in &labels {
            for term in label.split('-') {
                assert!(term.chars().count() >= 3, "short token {term:?} in {label:?}");
                assert!(!term.chars().all(|c| c.is_ascii_digit()), "numeric token {term:?} in {label:?}");
                assert!(!term.starts_with("0x"), "hex token {term:?} in {label:?}");
            }
        }
        assert!(labels[0].contains("syscall") || labels[0].contains("reserved") || labels[0].contains("zend"), "got {:?}", labels[0]);
        assert!(labels[1].contains("malloc") || labels[1].contains("struct"), "got {:?}", labels[1]);
    }

    #[test]
    fn meaningful_term_keeps_short_tech_terms() {
        // real 3-char tech terms must survive (tcp, bgp, i2p, gan)
        for t in ["tcp", "bgp", "i2p", "gan", "syscall", "malloc"] {
            assert!(is_meaningful_term(t), "{t} should be kept");
        }
        for t in ["0", "1", "i", "td", "br", "s", "0x0008", "2024", "42"] {
            assert!(!is_meaningful_term(t), "{t} should be dropped");
        }
    }

    #[test]
    fn assign_topics_nearest_with_lowest_index_tiebreak() {
        let centroids = vec![vec![1.0, 0.0], vec![0.0, 1.0]];
        let vecs = vec![
            vec![1.0, 0.0],
            vec![0.9, 0.1],
            vec![0.8, 0.2],
            vec![0.0, 1.0],
        ];
        let a = assign_topics(&centroids, &vecs);
        assert_eq!(a, vec![0, 0, 0, 1]);

        // counts 3:1 → largest share 0.75
        let mut counts = vec![0usize; centroids.len()];
        for &t in &a { counts[t] += 1; }
        let max_share = *counts.iter().max().unwrap() as f64 / a.len() as f64;
        assert!((max_share - 0.75).abs() < 1e-9, "got {max_share}");

        // a vector equidistant from both centroids ties to the lower index
        let tie = assign_topics(&centroids, &[vec![1.0, 1.0]]);
        assert_eq!(tie, vec![0]);
    }

    #[test]
    fn load_clusters_guards_model_and_emptiness() {
        let dir = std::env::temp_dir().join(format!("kibble_lc_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        let p = dir.join("clusters.json");

        let good = r#"{"k":2,"model":"nomic","centroids":[[1.0,0.0],[0.0,1.0]],"labels":["a","b"],"sizes":[1,1],"samples":["x","y"]}"#;
        std::fs::write(&p, good).unwrap();
        assert!(load_clusters(&p, "nomic").is_some());
        assert!(load_clusters(&p, "other").is_none(), "model mismatch must skip");
        assert!(load_clusters(&dir.join("missing.json"), "nomic").is_none(), "missing file must skip");

        let empty = r#"{"k":0,"model":"nomic","centroids":[],"labels":[],"sizes":[],"samples":[]}"#;
        std::fs::write(&p, empty).unwrap();
        assert!(load_clusters(&p, "nomic").is_none(), "empty centroids must skip");

        std::fs::remove_dir_all(&dir).ok();
    }

    #[test]
    fn write_clusters_round_trips() {
        let dir = std::env::temp_dir().join(format!("kibble_wc_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        let r = ClusterResult {
            k: 2, model: "nomic".into(),
            centroids: vec![vec![1.0, 0.0], vec![0.0, 1.0]],
            labels: vec!["a".into(), "b".into()],
            sizes: vec![3, 1], samples: vec!["x".into(), "y".into()],
        };
        write_clusters(&dir, "data/clusters.json", &r).unwrap();
        let back = load_clusters(&dir.join("data/clusters.json"), "nomic").unwrap();
        assert_eq!(back.centroids, r.centroids);
        assert_eq!(back.labels, r.labels);
        assert_eq!(back.sizes, r.sizes);
        std::fs::remove_dir_all(&dir).ok();
    }

    #[test]
    fn build_label_prompt_lists_topics() {
        let labels = vec!["old-css".to_string(), "quantum-physics".to_string()];
        let samples = vec!["the state of css centering".to_string(), "quantum entanglement basics".to_string()];
        let p = build_label_prompt(&labels, &samples);
        assert!(p.contains("Topic 1"), "got: {p}");
        assert!(p.contains("Topic 2"));
        assert!(p.contains("old, css"), "'-' should expand to ', ': {p}");
        assert!(p.contains("css centering"), "sample should appear: {p}");
        assert!(p.to_lowercase().contains("json array"));
        assert!(p.contains("exactly 2"), "prompt must demand the exact topic count: {p}");
    }

    #[test]
    fn parse_labels_valid_and_rejects() {
        assert_eq!(
            parse_labels(r#"["CSS Centering","Quantum Physics"]"#, 2),
            Some(vec!["CSS Centering".to_string(), "Quantum Physics".to_string()])
        );
        // fenced / prose-wrapped array still parses via bracket extraction
        assert_eq!(parse_labels("Sure!\n```json\n[\"A\",\"B\"]\n```", 2), Some(vec!["A".to_string(), "B".to_string()]));
        assert!(parse_labels(r#"["only one"]"#, 2).is_none(), "wrong count → None");
        assert!(parse_labels("no array here", 2).is_none(), "no brackets → None");
        assert!(parse_labels(r#"["ok",""]"#, 2).is_none(), "empty name → None");
        let long = "x".repeat(80);
        let r = parse_labels(&format!("[\"{long}\"]"), 1).unwrap();
        assert_eq!(r[0].chars().count(), 48, "over-long name capped to 48");
    }

    #[test]
    fn hier_prompt_mentions_path_format() {
        let p = build_hier_label_prompt(&["css-grid".to_string()], &["a sample about grid".to_string()]);
        assert!(p.contains(HIERARCHICAL_SEPARATOR), "prompt shows the path separator");
        assert!(p.to_lowercase().contains("categor"), "prompt asks for categories/hierarchy");
        assert!(p.contains("exactly 1"), "prompt must demand the exact topic count: {p}");
    }

    #[test]
    fn parse_labels_reads_hierarchical_paths() {
        let resp = "[\"Web › Frontend › CSS\", \"Systems › Linux\"]";
        let got = parse_labels(resp, 2).unwrap();
        assert_eq!(got, vec!["Web › Frontend › CSS".to_string(), "Systems › Linux".to_string()]);
    }

    #[tokio::test]
    async fn apply_llm_labels_failsoft_noop() {
        let dir = std::env::temp_dir().join(format!("kibble_llml_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        let mk = || ClusterResult {
            k: 2, model: "m".into(),
            centroids: vec![vec![1.0], vec![0.0]],
            labels: vec!["term-a".into(), "term-b".into()],
            sizes: vec![1, 1], samples: vec!["x".into(), "y".into()],
        };
        let term = vec!["term-a".to_string(), "term-b".to_string()];

        // flag off (default) → labels unchanged
        std::fs::write(dir.join(crate::config::CONFIG_FILE), "").unwrap();
        let cfg = crate::config::load_config(&dir.join(crate::config::CONFIG_FILE));
        let mut r = mk();
        apply_llm_labels(&cfg, &mut r).await;
        assert_eq!(r.labels, term);

        // flag on but empty [ask].base_url → unchanged (returns before any network)
        std::fs::write(dir.join(crate::config::CONFIG_FILE), "[cluster]\nllm_labels=true\n[ask]\nbase_url=\"\"\n").unwrap();
        let cfg = crate::config::load_config(&dir.join(crate::config::CONFIG_FILE));
        let mut r = mk();
        apply_llm_labels(&cfg, &mut r).await;
        assert_eq!(r.labels, term);

        std::fs::remove_dir_all(&dir).ok();
    }
}