glossia 0.2.0

use std::collections::{HashMap, HashSet};
#[cfg(not(target_arch = "wasm32"))]
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock};
#[cfg(not(target_arch = "wasm32"))]
use std::time::UNIX_EPOCH;
#[cfg(not(target_arch = "wasm32"))]
use std::collections::hash_map::DefaultHasher;
#[cfg(not(target_arch = "wasm32"))]
use std::hash::{Hash, Hasher};
#[cfg(not(target_arch = "wasm32"))]
use std::env;
use rand::{seq::SliceRandom, Rng};
use crate::types::Pos;
use crate::merkle::WordlistTree;

/// Resolve payload and cover filenames for a given wordlist profile.
/// Returns (payload_filename, cover_filename).
/// - `"default"` → `payload.yaml` / `cover.yaml` (every language's base wordlist)
/// - `"bip39"`   → `payload_bip39.yaml` / `cover.yaml` (English-specific BIP39 list)
/// - other       → `payload_{name}.yaml` / `cover_{name}.yaml` (falls back to `cover.yaml`)
pub fn wordlist_filenames(language: &str, wordlist: &str) -> (String, String) {
    match wordlist {
        "default" => ("payload.yaml".into(), "cover.yaml".into()),
        other => {
            let payload = format!("payload_{}.yaml", other);
            let cover = format!("cover_{}.yaml", other);
            let cover_key = format!("{}/{}", language, cover);
            // Check embedded YAML first (release builds have all languages)
            if get_embedded_yaml(&cover_key).is_some() {
                return (payload, cover);
            }
            // In debug builds, non-English languages aren't embedded.
            // Fall back to filesystem check for the cover file.
            #[cfg(not(target_arch = "wasm32"))]
            {
                if find_language_file(language, &cover).is_some() {
                    return (payload, cover);
                }
            }
            (payload, "cover.yaml".into())
        }
    }
}

// Include auto-generated language index
#[allow(dead_code)]
mod language_index {
    include!(concat!(env!("OUT_DIR"), "/language_index.rs"));
}

/// Get embedded YAML file content by path relative to languages/ directory.
/// Returns Some(content) if file is embedded in release builds, None otherwise.
/// Path should be like "english/payload.yaml" or "latin/grammar.yaml"
/// This function is now auto-generated by build.rs
pub fn get_embedded_yaml(path: &str) -> Option<&'static str> {
    language_index::get_embedded_yaml(path)
}

/// Check if a language has embedded files (packaged with the binary)
/// In release builds, all languages with YAML files are embedded.
/// In debug builds, only English is embedded (for faster iteration).
/// This function is now auto-generated by build.rs
pub fn has_embedded_files(language: &str) -> bool {
    language_index::has_embedded_files(language)
}

/// Load the optional semantic model (`<language>/semantics.yaml`) used to softly
/// bias sentence planning toward coherent verb-argument pairings. Returns `None`
/// when the language ships no such file (the common case), leaving generation
/// behavior unchanged. Never affects decoding.
pub fn load_semantics(language: &str) -> Option<crate::generator::semantics::SemanticModel> {
    // Escape hatch / A-B switch: GLOSSIA_DISABLE_SEMANTICS=1 forces the classic
    // POS-only planning behavior. Never affects decoding either way.
    #[cfg(not(target_arch = "wasm32"))]
    if std::env::var("GLOSSIA_DISABLE_SEMANTICS").is_ok() {
        return None;
    }
    let content = get_embedded_yaml(&format!("{}/semantics.yaml", language))?;
    match crate::generator::semantics::SemanticModel::from_yaml(content) {
        Ok(model) if !model.is_empty() => Some(model),
        Ok(_) => None,
        Err(e) => {
            #[cfg(not(target_arch = "wasm32"))]
            eprintln!("warning: ignoring {}/semantics.yaml: {}", language, e);
            let _ = e;
            None
        }
    }
}

/// Get the list of available languages (auto-generated by build.rs)
pub fn get_available_languages() -> &'static [&'static str] {
    language_index::get_available_languages()
}

/// Get available wordlist profiles for a language.
/// Derived at compile time from payload filenames in the languages/ directory.
pub fn get_available_wordlists(language: &str) -> Vec<String> {
    language_index::get_wordlist_profiles(language)
        .iter()
        .map(|s| s.to_string())
        .collect()
}

/// Get the default wordlist profile for a language.
///
/// Resolution order:
/// 1. Grammar-declared `default_wordlist` field (explicit, from grammar.yaml)
/// 2. First profile from `get_wordlist_profiles()` (build-time alphabetical fallback)
/// 3. `"default"` (payload.yaml)
pub fn default_wordlist(language: &str) -> &'static str {
    if let Some(dw) = language_index::get_grammar_default_wordlist(language) {
        return dw;
    }
    let profiles = language_index::get_wordlist_profiles(language);
    profiles.first().copied().unwrap_or("default")
}

/// Get the exact precomputed word count for a payload wordlist.
/// Returns 0 if the language/wordlist combination is not found.
pub fn get_wordlist_size(language: &str, wordlist: &str) -> usize {
    language_index::get_payload_word_count(language, wordlist)
}

#[derive(serde::Serialize, serde::Deserialize)]
struct PayloadCacheData {
    /// Sorted list of all payload words for the language.
    words: Vec<String>,
    /// Mapping from payload word -> allowed POS tags derived from payload.yaml (+ optional pos_mapping.yaml).
    pos_mapping: HashMap<String, Vec<Pos>>,
}

static PAYLOAD_CACHE_BY_LANGUAGE: OnceLock<Mutex<HashMap<String, Arc<PayloadCacheData>>>> = OnceLock::new();

#[cfg(not(target_arch = "wasm32"))]
fn glossia_cache_dir() -> PathBuf {
    if let Some(p) = std::env::var_os("GLOSSIA_CACHE_DIR") {
        return PathBuf::from(p);
    }
    let base = std::env::var_os("XDG_CACHE_HOME")
        .map(PathBuf::from)
        .or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".cache")))
        .unwrap_or_else(std::env::temp_dir);
    base.join("glossia")
}

#[cfg(not(target_arch = "wasm32"))]
fn file_fingerprint(path: &Path) -> Option<(u64, u64, u32)> {
    let md = std::fs::metadata(path).ok()?;
    let modified = md.modified().ok()?;
    let dur = modified.duration_since(UNIX_EPOCH).ok()?;
    Some((md.len(), dur.as_secs(), dur.subsec_nanos()))
}

#[cfg(not(target_arch = "wasm32"))]
fn cache_key_u64_for_language(language: &str, wordlist: &str, payload_path: Option<&Path>, pos_mapping_path: Option<&Path>) -> u64 {
    let mut hasher = DefaultHasher::new();
    language.hash(&mut hasher);
    wordlist.hash(&mut hasher);
    env!("CARGO_PKG_VERSION").hash(&mut hasher);

    let (payload_filename, _) = wordlist_filenames(language, wordlist);
    if let Some(p) = payload_path {
        p.to_string_lossy().hash(&mut hasher);
        file_fingerprint(p).hash(&mut hasher);
    } else {
        // Embedded payload. Hashing bytes is cheap vs YAML parsing and
        // keeps the cache stable across runs while auto-invalidating when the embedded data changes.
        if let Some(embedded_payload) = get_embedded_yaml(&format!("{}/{}", language, payload_filename)) {
            embedded_payload.as_bytes().hash(&mut hasher);
        }
    }

    if let Some(p) = pos_mapping_path {
        p.to_string_lossy().hash(&mut hasher);
        file_fingerprint(p).hash(&mut hasher);
    }

    hasher.finish()
}

#[cfg(not(target_arch = "wasm32"))]
fn payload_cache_file_path(language: &str, wordlist: &str, payload_path: Option<&Path>, pos_mapping_path: Option<&Path>) -> PathBuf {
    let key = cache_key_u64_for_language(language, wordlist, payload_path, pos_mapping_path);
    glossia_cache_dir().join(format!("payload_cache_{language}_{wordlist}_{key:016x}.bin"))
}

fn load_or_build_payload_cache(language: &str, wordlist: &str) -> Result<Arc<PayloadCacheData>, String> {
    // In-memory cache keyed on (language, wordlist).
    let cache_key = format!("{}:{}", language, wordlist);
    let cache_map = PAYLOAD_CACHE_BY_LANGUAGE.get_or_init(|| Mutex::new(HashMap::new()));
    if let Some(existing) = cache_map.lock().unwrap().get(&cache_key).cloned() {
        return Ok(existing);
    }

    let (payload_filename, _) = wordlist_filenames(language, wordlist);

    // Determine file paths (if any) for on-disk cache keying/invalidation.
    // On WASM, we always use embedded files (no filesystem).
    #[cfg(not(target_arch = "wasm32"))]
    let payload_path: Option<PathBuf> = if has_embedded_files(language) {
        None
    } else {
        Some(PathBuf::from(get_wordlist_path(language, wordlist)?))
    };
    #[cfg(not(target_arch = "wasm32"))]
    let pos_mapping_path: Option<PathBuf> = payload_path
        .as_ref()
        .and_then(|p| p.parent().map(|d| d.join("pos_mapping.yaml")))
        .filter(|p| p.exists());

    // Try on-disk cache (native only).
    #[cfg(not(target_arch = "wasm32"))]
    {
        let cache_file = payload_cache_file_path(language, wordlist, payload_path.as_deref(), pos_mapping_path.as_deref());
        if let Ok(bytes) = std::fs::read(&cache_file) {
            if let Ok(data) = bincode::deserialize::<PayloadCacheData>(&bytes) {
                let arc = Arc::new(data);
                cache_map.lock().unwrap().insert(cache_key.clone(), arc.clone());
                return Ok(arc);
            }
        }
    }

    // Build from YAML (single parse for both wordlist + POS mapping).
    // Try embedded YAML for this language first, then fall back to cs/ (shared character-level
    // wordlists like bech32, base58), then filesystem.
    let yaml_content = if let Some(embedded) = get_embedded_yaml(&format!("{}/{}", language, payload_filename)) {
        embedded.to_string()
    } else if language != "cs" {
        if let Some(embedded) = get_embedded_yaml(&format!("cs/{}", payload_filename)) {
            embedded.to_string()
        } else {
            #[cfg(not(target_arch = "wasm32"))]
            {
                // Try filesystem: first in this language's dir, then in cs/
                if let Some(ref p) = payload_path {
                    std::fs::read_to_string(p).map_err(|e| format!("Failed to read YAML file '{}': {}", p.display(), e))?
                } else if let Some(cs_path) = find_language_file("cs", &payload_filename) {
                    std::fs::read_to_string(&cs_path).map_err(|e| format!("Failed to read YAML file '{}': {}", cs_path, e))?
                } else {
                    return Err(format!("No payload YAML for language '{}', wordlist '{}' (also tried cs/)", language, wordlist));
                }
            }
            #[cfg(target_arch = "wasm32")]
            {
                return Err(format!("No embedded YAML for language '{}', wordlist '{}'", language, wordlist));
            }
        }
    } else {
        #[cfg(not(target_arch = "wasm32"))]
        {
            let p = payload_path.as_ref().expect("payload_path must exist for non-embedded languages");
            std::fs::read_to_string(p).map_err(|e| format!("Failed to read YAML file '{}': {}", p.display(), e))?
        }
        #[cfg(target_arch = "wasm32")]
        {
            return Err(format!("No embedded YAML for language '{}', wordlist '{}'", language, wordlist));
        }
    };

    // Parse with serde_yaml::Value to preserve YAML key order.
    // Word ordering in the YAML file is authoritative for codec indices.
    // serde_yaml 0.9 Mapping uses IndexMap internally, so iteration order = file order.
    use serde_yaml::Value;
    let yaml_value: Value = serde_yaml::from_str(&yaml_content)
        .map_err(|e| format!("Failed to parse YAML: {}", e))?;

    let mapping = yaml_value.as_mapping()
        .ok_or_else(|| "Payload YAML is not a mapping".to_string())?;

    // Load language-specific POS tag mappings (small file; only used when building cache).
    let pos_mappings = load_pos_mappings(language);

    let mut words: Vec<String> = Vec::with_capacity(mapping.len());
    let mut pos_mapping: HashMap<String, Vec<Pos>> = HashMap::with_capacity(mapping.len());
    let mut seen: std::collections::HashSet<String> = std::collections::HashSet::with_capacity(mapping.len());

    for (key, value) in mapping {
        let word = match key.as_str() {
            Some(s) => s.to_lowercase(),
            None => continue,
        };
        if word.is_empty() || !seen.insert(word.clone()) {
            continue;
        }

        words.push(word.clone());

        let mut pos_tags = Vec::new();
        if let Some(pos_map) = value.as_mapping() {
            for (pos_key, pos_val) in pos_map {
                if let (Some(pos_str), Some(weight)) = (pos_key.as_str(), pos_val.as_f64()) {
                    if weight > 0.0 {
                        if let Some(pos) = parse_pos_tag(pos_str, &pos_mappings) {
                            pos_tags.push(pos);
                        }
                    }
                }
            }
        }

        if !pos_tags.is_empty() {
            pos_mapping.insert(word, pos_tags);
        }
    }

    let data = PayloadCacheData { words, pos_mapping };

    // Best-effort on-disk cache write (native only).
    #[cfg(not(target_arch = "wasm32"))]
    {
        let cache_file = payload_cache_file_path(language, wordlist, payload_path.as_deref(), pos_mapping_path.as_deref());
        let cache_dir = glossia_cache_dir();
        let _ = std::fs::create_dir_all(&cache_dir);
        if let Ok(bytes) = bincode::serialize(&data) {
            let _ = std::fs::write(&cache_file, bytes);
        }
    }

    let arc = Arc::new(data);
    cache_map.lock().unwrap().insert(cache_key, arc.clone());
    Ok(arc)
}

/// Load language-specific POS tag mappings from pos_mapping.yaml
/// Returns a HashMap mapping language-specific POS tag names to Glossia POS tags
/// Returns empty HashMap if file doesn't exist (backward compatible)
fn load_pos_mappings(language: &str) -> HashMap<String, Pos> {
    // Try embedded file first (release builds)
    let yaml_content = if let Some(embedded) = get_embedded_yaml(&format!("{}/pos_mapping.yaml", language)) {
        embedded.to_string()
    } else {
        #[cfg(not(target_arch = "wasm32"))]
        {
            // Fall back to filesystem lookup (for debug builds or languages without embedded files)
            if let Some(pos_mapping_path) = find_language_file(language, "pos_mapping.yaml") {
                match std::fs::read_to_string(&pos_mapping_path) {
                    Ok(content) => content,
                    Err(_) => return HashMap::new(),
                }
            } else {
                return HashMap::new();
            }
        }
        #[cfg(target_arch = "wasm32")]
        {
            return HashMap::new();
        }
    };

    // Parse YAML structure: { mappings: { language_pos: glossia_pos, ... } }
    #[derive(serde::Deserialize)]
    struct PosMappingFile {
        mappings: Option<HashMap<String, String>>,
    }

    let mapping_file: PosMappingFile = match serde_yaml::from_str(&yaml_content) {
        Ok(m) => m,
        Err(_) => return HashMap::new(),
    };

    let mut result = HashMap::new();
    if let Some(mappings) = mapping_file.mappings {
        for (lang_pos, glossia_pos_str) in mappings {
            if let Some(pos) = Pos::from_str(&glossia_pos_str) {
                result.insert(lang_pos, pos);
            }
        }
    }

    result
}

/// Build comprehensive POS mapping for all payload words.
/// Returns a HashMap mapping each word to its allowed POS tags.
/// Uses YAML format (word -> POS weights) from payload.yaml.
pub fn build_pos_mapping(language: &str) -> Result<HashMap<String, Vec<Pos>>, String> {
    build_pos_mapping_for_wordlist(language, default_wordlist(language))
}

/// Build POS mapping for a specific wordlist profile.
pub fn build_pos_mapping_for_wordlist(language: &str, wordlist: &str) -> Result<HashMap<String, Vec<Pos>>, String> {
    // Use shared payload cache (parses payload.yaml once; subsequent calls are near-zero cost).
    Ok(load_or_build_payload_cache(language, wordlist)?.pos_mapping.clone())
}

/// Load POS mapping from YAML file on disk
#[cfg(not(target_arch = "wasm32"))]
pub fn build_pos_mapping_from_yaml(path: &str, pos_mappings: &HashMap<String, Pos>) -> Result<HashMap<String, Vec<Pos>>, String> {
    let yaml_content = std::fs::read_to_string(path)
        .map_err(|e| format!("Failed to read YAML file '{}': {}", path, e))?;
    build_pos_mapping_from_yaml_content(&yaml_content, pos_mappings)
}

/// Build POS mapping from YAML format.
/// YAML structure: { word: { POS: weight, ... }, ... }
/// Weights are normalized and POS tags are extracted.
pub fn build_pos_mapping_from_yaml_content(yaml_content: &str, pos_mappings: &HashMap<String, Pos>) -> Result<HashMap<String, Vec<Pos>>, String> {
    use serde_yaml::Value;
    let yaml_data: HashMap<String, HashMap<String, Value>> = serde_yaml::from_str(yaml_content)
        .map_err(|e| format!("Failed to parse YAML: {}", e))?;

    let mut mapping = HashMap::new();

    for (word, pos_weights) in yaml_data {
        let word_lower = word.to_lowercase();
        let mut pos_tags = Vec::new();

        for (pos_str, value) in pos_weights {
            if let Some(weight) = value.as_f64() {
                if weight > 0.0 {
                    if let Some(pos) = parse_pos_tag(&pos_str, pos_mappings) {
                        pos_tags.push(pos);
                    }
                }
            }
        }

        if !word_lower.is_empty() && !pos_tags.is_empty() {
            mapping.insert(word_lower, pos_tags);
        }
    }

    Ok(mapping)
}

/// Parse a single POS tag string to Pos enum.
/// First checks language-specific mappings, then falls back to default POS tags.
pub fn parse_pos_tag(pos_str: &str, pos_mappings: &HashMap<String, Pos>) -> Option<Pos> {
    let pos_str_trimmed = pos_str.trim();

    // First check language-specific mappings
    if let Some(pos) = pos_mappings.get(pos_str_trimmed) {
        return Some(*pos);
    }

    // Fall back to default POS tags
    Pos::from_str(pos_str_trimmed)
}

/// Get POS tags for a word from the comprehensive mapping.
/// Returns a vector of allowed POS tags.
/// Used for tagging payload (BIP39) words. Cover words use explicit POS tags from cover.yaml.
pub fn tag_word(word: &str) -> Vec<Pos> {
    static POS_MAP: OnceLock<HashMap<String, Vec<Pos>>> = OnceLock::new();

    let mapping = POS_MAP.get_or_init(|| {
        build_pos_mapping("english").unwrap_or_else(|_| HashMap::new())
    });

    let word_lower = word.to_lowercase();
    mapping.get(&word_lower).cloned().unwrap_or_default()
}

/// Load all payload words from the default wordlist file.
/// Uses YAML format (extracts keys) from payload.yaml.
/// For languages with embedded files, uses embedded file. For other languages, tries to load from filesystem.
pub fn load_payload_words(language: &str) -> Result<Vec<String>, String> {
    load_payload_words_for_wordlist(language, default_wordlist(language))
}

/// Load payload words for a specific wordlist profile.
pub fn load_payload_words_for_wordlist(language: &str, wordlist: &str) -> Result<Vec<String>, String> {
    Ok(load_or_build_payload_cache(language, wordlist)?.words.clone())
}

/// Inject a scale-derived payload wordlist into the in-memory cache.
///
/// Call this before any `load_payload_words_for_wordlist()` for a scale-based dialect.
/// The words are derived from the chromatic payload filtered by a scale definition,
/// so no `payload_*.yaml` file is needed.
///
/// All words are tagged with POS `N` (the only payload POS in the music language).
pub fn inject_scale_payload(language: &str, wordlist_name: &str, mut words: Vec<String>) -> Result<(), String> {
    let cache_key = format!("{}:{}", language, wordlist_name);
    let cache_map = PAYLOAD_CACHE_BY_LANGUAGE.get_or_init(|| Mutex::new(HashMap::new()));

    // Build POS mapping: every scale note is N (noun = pitch predicate).
    let mut pos_mapping: HashMap<String, Vec<Pos>> = HashMap::with_capacity(words.len());
    for word in &words {
        pos_mapping.insert(word.to_lowercase(), vec![Pos::N]);
    }

    words.sort();
    words.dedup();

    let data = PayloadCacheData { words, pos_mapping };
    cache_map.lock().unwrap().insert(cache_key, Arc::new(data));
    Ok(())
}

/// Load payload words from embedded payload.yaml
pub fn load_payload_words_from_embedded(language: &str) -> Result<Vec<String>, String> {
    let (payload_filename, _) = wordlist_filenames(language, default_wordlist(language));
    let payload_yaml = get_embedded_yaml(&format!("{}/{}", language, payload_filename))
        .ok_or_else(|| format!("No embedded file for language: {}", language))?;
    load_payload_words_from_yaml_content(payload_yaml)
}

/// Load payload words from YAML content string (extracts keys in file order).
pub fn load_payload_words_from_yaml_content(yaml_content: &str) -> Result<Vec<String>, String> {
    use serde_yaml::Value;
    let yaml_value: Value = serde_yaml::from_str(yaml_content)
        .map_err(|e| format!("Failed to parse YAML: {}", e))?;

    let mapping = yaml_value.as_mapping()
        .ok_or_else(|| "Payload YAML is not a mapping".to_string())?;

    let mut words: Vec<String> = Vec::with_capacity(mapping.len());
    let mut seen: std::collections::HashSet<String> = std::collections::HashSet::with_capacity(mapping.len());
    for (key, _) in mapping {
        if let Some(s) = key.as_str() {
            let word = s.to_lowercase();
            if !word.is_empty() && seen.insert(word.clone()) {
                words.push(word);
            }
        }
    }
    Ok(words)
}

/// Load payload words from YAML file (extracts keys).
#[cfg(not(target_arch = "wasm32"))]
pub fn load_payload_words_from_yaml(path: &str) -> Result<Vec<String>, String> {
    let yaml_content = std::fs::read_to_string(path)
        .map_err(|e| format!("Failed to read YAML file '{}': {}", path, e))?;
    load_payload_words_from_yaml_content(&yaml_content)
}

/// Load payload words into a WordlistTree.
/// Returns a WordlistTree with canonical ordering (sorted alphabetically for payload words).
pub fn load_payload_tree(language: &str) -> Result<WordlistTree, String> {
    let words = load_payload_words(language)?;
    Ok(WordlistTree::new(words))
}

/// Load cover words with POS tags from cover.yaml
/// Returns a HashMap mapping POS to Vec of words
/// For English, uses embedded file. For other languages, tries to load from filesystem.
pub fn load_cover_words_by_pos(wordlist_set: &HashSet<String>, language: &str) -> (HashMap<Pos, Vec<String>>, HashMap<(Pos, String), Vec<String>>) {
    load_cover_words_by_pos_for_wordlist(wordlist_set, language, default_wordlist(language))
}

/// Load cover words for a specific wordlist profile.
/// Returns (by_pos, refined_cover) where refined_cover maps (POS, refinement_tag) -> words.
pub fn load_cover_words_by_pos_for_wordlist(wordlist_set: &HashSet<String>, language: &str, wordlist: &str) -> (HashMap<Pos, Vec<String>>, HashMap<(Pos, String), Vec<String>>) {
    let (_, cover_filename) = wordlist_filenames(language, wordlist);
    // Load language-specific POS tag mappings
    let pos_mappings = load_pos_mappings(language);

    let yaml_content = if let Some(embedded) = get_embedded_yaml(&format!("{}/{}", language, cover_filename)) {
        embedded.to_string()
    } else {
        #[cfg(not(target_arch = "wasm32"))]
        {
            // For other languages, try to load from filesystem (with recursive search)
            let cover_yaml_path = find_language_file(language, &cover_filename)
                .unwrap_or_else(|| {
                    let languages_dir = find_languages_dir()
                        .unwrap_or_else(|| "languages".to_string());
                    format!("{}/{}/{}", languages_dir, language, cover_filename)
                });
            std::fs::read_to_string(&cover_yaml_path)
                .unwrap_or_else(|e| {
                    panic!("Error: Failed to read {} from '{}': {}", cover_filename, cover_yaml_path, e);
                })
        }
        #[cfg(target_arch = "wasm32")]
        {
            panic!("No embedded cover file for language '{}', wordlist '{}'", language, wordlist);
        }
    };

    // Parse with serde_yaml::Value to handle mixed types (f64 weights + string refinement tag)
    use serde_yaml::Value;
    let yaml_data: HashMap<String, Value> = serde_yaml::from_str(&yaml_content)
        .unwrap_or_else(|e| {
            panic!("Error: Failed to parse cover.yaml as YAML: {}", e);
        });

    let mut by_pos: HashMap<Pos, Vec<String>> = HashMap::new();
    let mut refined_cover: HashMap<(Pos, String), Vec<String>> = HashMap::new();

    for (word, value) in &yaml_data {
        let word_lower = word.to_lowercase();

        // Skip if word is in wordlist set
        if wordlist_set.contains(&word_lower) {
            continue;
        }

        let mapping = match value.as_mapping() {
            Some(m) => m,
            None => continue,
        };

        // Extract refinement tag (if present)
        let refinement = mapping.get("refinement")
            .and_then(|v| v.as_str())
            .map(|s| s.to_string());

        // Extract POS tags with non-zero weights (skip "refinement" key)
        for (key, val) in mapping {
            let pos_str = match key.as_str() {
                Some(s) if s != "refinement" => s,
                _ => continue,
            };
            let weight = match val.as_f64() {
                Some(w) if w > 0.0 => w,
                _ => continue,
            };

            if weight > 0.0 {
                if let Some(pos) = parse_pos_tag(pos_str, &pos_mappings) {
                    by_pos.entry(pos).or_insert_with(Vec::new).push(word.clone());

                    // Also index by (POS, refinement) if refinement is present
                    if let Some(ref tag) = refinement {
                        refined_cover.entry((pos, tag.clone()))
                            .or_insert_with(Vec::new)
                            .push(word.clone());
                    }
                }
            }
        }
    }

    // Deduplicate and sort each category
    for words in by_pos.values_mut() {
        words.sort();
        words.dedup();
    }
    for words in refined_cover.values_mut() {
        words.sort();
        words.dedup();
    }

    (by_pos, refined_cover)
}

/// Load cover words in file order from cover.yaml.
/// Returns all cover words preserving the order from cover.yaml (never re-sorted; this IS the canonical ordering).
/// Key invariant: The file order of cover.yaml is the canonical ordering. Code must never sort this list.
pub fn load_cover_words_in_file_order(language: &str) -> Vec<String> {
    load_cover_words_in_file_order_for_wordlist(language, default_wordlist(language))
}

/// Load cover words in file order for a specific wordlist profile.
pub fn load_cover_words_in_file_order_for_wordlist(language: &str, wordlist: &str) -> Vec<String> {
    let (_, cover_filename) = wordlist_filenames(language, wordlist);
    let yaml_content = if let Some(embedded) = get_embedded_yaml(&format!("{}/{}", language, cover_filename)) {
        embedded.to_string()
    } else {
        #[cfg(not(target_arch = "wasm32"))]
        {
            // For other languages, try to load from filesystem (with recursive search)
            let cover_yaml_path = find_language_file(language, &cover_filename)
                .unwrap_or_else(|| {
                    let languages_dir = find_languages_dir()
                        .unwrap_or_else(|| "languages".to_string());
                    format!("{}/{}/{}", languages_dir, language, cover_filename)
                });
            std::fs::read_to_string(&cover_yaml_path)
                .unwrap_or_else(|e| {
                    panic!("Error: Failed to read {} from '{}': {}", cover_filename, cover_yaml_path, e);
                })
        }
        #[cfg(target_arch = "wasm32")]
        {
            panic!("No embedded cover file for language '{}', wordlist '{}'", language, wordlist);
        }
    };

    use serde_yaml::Value;
    let value: Value = serde_yaml::from_str(&yaml_content)
        .unwrap_or_else(|e| {
            panic!("Error: Failed to parse cover.yaml as YAML Value: {}", e);
        });

    let mut words = Vec::new();
    if let Value::Mapping(mapping) = value {
        // serde_yaml::Mapping preserves insertion order
        for (key, _) in mapping {
            if let Value::String(word) = key {
                words.push(word);
            }
        }
    }

    words
}

/// Load cover words into a WordlistTree.
/// Returns a WordlistTree preserving file order from cover.yaml (canonical ordering).
/// Key invariant: The file order of cover.yaml is the canonical ordering. Code must never sort this list.
pub fn load_cover_tree(language: &str) -> WordlistTree {
    let words = load_cover_words_in_file_order(language);
    WordlistTree::new(words)
}

/// Load POS tags for cover words from cover.yaml.
/// Returns a HashMap mapping cover word -> Vec of POS tags.
/// This allows Merkle words to be treated as PayloadToks with proper POS.
pub fn load_cover_word_pos_tags(language: &str) -> HashMap<String, Vec<Pos>> {
    load_cover_word_pos_tags_for_wordlist(language, "default")
}

/// Load cover word POS tags for a specific wordlist profile.
pub fn load_cover_word_pos_tags_for_wordlist(language: &str, wordlist: &str) -> HashMap<String, Vec<Pos>> {
    let (_, cover_filename) = wordlist_filenames(language, wordlist);
    // Load language-specific POS tag mappings
    let pos_mappings = load_pos_mappings(language);

    let yaml_content = if let Some(embedded) = get_embedded_yaml(&format!("{}/{}", language, cover_filename)) {
        embedded.to_string()
    } else {
        #[cfg(not(target_arch = "wasm32"))]
        {
            // For other languages, try to load from filesystem (with recursive search)
            let cover_yaml_path = find_language_file(language, &cover_filename)
                .unwrap_or_else(|| {
                    let languages_dir = find_languages_dir()
                        .unwrap_or_else(|| "languages".to_string());
                    format!("{}/{}/{}", languages_dir, language, cover_filename)
                });
            std::fs::read_to_string(&cover_yaml_path)
                .unwrap_or_else(|e| {
                    panic!("Error: Failed to read {} from '{}': {}", cover_filename, cover_yaml_path, e);
                })
        }
        #[cfg(target_arch = "wasm32")]
        {
            panic!("No embedded cover file for language '{}', wordlist '{}'", language, wordlist);
        }
    };

    use serde_yaml::Value;
    let yaml_data: HashMap<String, HashMap<String, Value>> = serde_yaml::from_str(&yaml_content)
        .unwrap_or_else(|e| {
            panic!("Error: Failed to parse cover.yaml as YAML: {}", e);
        });

    let mut result: HashMap<String, Vec<Pos>> = HashMap::new();

    for (word, pos_weights) in yaml_data {
        let word_lower = word.to_lowercase();
        let mut pos_tags = Vec::new();

        // Extract POS tags with non-zero weights
        for (pos_str, value) in pos_weights {
            let weight = value.as_f64().unwrap_or(0.0);
            if weight > 0.0 {
                if let Some(pos) = parse_pos_tag(&pos_str, &pos_mappings) {
                    pos_tags.push(pos);
                }
            }
        }

        if !pos_tags.is_empty() {
            result.insert(word_lower, pos_tags);
        }
    }

    result
}

/// Randomly select N words from the BIP39 wordlist.
pub fn select_random_words<R: Rng>(rng: &mut R, count: usize, language: &str) -> Result<Vec<String>, String> {
    let all_words = load_payload_words(language)?;
    if all_words.is_empty() || count == 0 {
        return Ok(Vec::new());
    }

    // Sample WITH replacement so duplicates are possible (and therefore decodable).
    let mut selected = Vec::with_capacity(count);
    for _ in 0..count {
        selected.push(all_words.choose(rng).unwrap().clone());
    }
    Ok(selected)
}

/// Find the languages directory by checking multiple locations
#[cfg(not(target_arch = "wasm32"))]
fn find_languages_dir() -> Option<String> {
    let probe = "languages/english/payload_bip39.yaml";
    // Try current directory first (for development)
    if std::path::Path::new(probe).exists() {
        return Some("languages".to_string());
    }

    // Walk up parent directories from the current directory. This handles being
    // run from a workspace member subdirectory (e.g. `cargo test -p glossia-cli`,
    // whose test binary runs with CWD set to the member crate root) while the
    // `languages/` folder lives at the workspace root one or more levels up.
    if let Ok(mut dir) = std::env::current_dir() {
        loop {
            let candidate = dir.join("languages");
            if candidate.join("english/payload_bip39.yaml").exists() {
                return Some(candidate.to_string_lossy().to_string());
            }
            if !dir.pop() {
                break;
            }
        }
    }

    // Try relative to the binary location (for installed binaries)
    if let Ok(exe_path) = std::env::current_exe() {
        if let Some(exe_dir) = exe_path.parent() {
            // Check if languages dir is sibling to bin dir (cargo install layout)
            let languages_path = exe_dir.join("../share/glossia/languages");
            if languages_path.join("english/payload_bip39.yaml").exists() {
                return Some(languages_path.to_string_lossy().to_string());
            }

            // Check if languages dir is in the same directory as binary
            let languages_path = exe_dir.join("languages");
            if languages_path.join("english/payload_bip39.yaml").exists() {
                return Some(languages_path.to_string_lossy().to_string());
            }

            // Check if we're in a cargo install location, go up to find languages
            if exe_dir.ends_with("bin") {
                let share_path = exe_dir.join("../share/glossia/languages");
                if share_path.join("english/payload_bip39.yaml").exists() {
                    return Some(share_path.to_string_lossy().to_string());
                }
            }
        }
    }

    // Try home directory cargo share location
    if let Some(home) = std::env::var_os("HOME") {
        let cargo_share = std::path::Path::new(&home).join(".cargo/share/glossia/languages");
        if cargo_share.join("english/payload_bip39.yaml").exists() {
            return Some(cargo_share.to_string_lossy().to_string());
        }
    }

    // Try CARGO_HOME if set
    if let Ok(cargo_home) = std::env::var("CARGO_HOME") {
        let cargo_share = std::path::Path::new(&cargo_home).join("share/glossia/languages");
        if cargo_share.join("english/payload_bip39.yaml").exists() {
            return Some(cargo_share.to_string_lossy().to_string());
        }
    }

    None
}

/// Exits with error if payload file doesn't exist.
/// Only used for languages without embedded files (those use embedded files instead).
/// Supports subdirectories: "math/primes" -> "languages/math/primes/payload.yaml"
#[cfg(not(target_arch = "wasm32"))]
pub fn get_wordlist_path(language: &str, wordlist: &str) -> Result<String, String> {
    // Languages with embedded files should not call this function
    if has_embedded_files(language) {
        return Err(format!("Language '{}' uses embedded files and should not call get_wordlist_path", language));
    }

    let (payload_filename, _) = wordlist_filenames(language, wordlist);

    // Try to find languages directory
    let languages_dir = find_languages_dir()
        .ok_or_else(|| format!("Could not find languages directory. Please ensure the 'languages' folder is accessible.\nTried: current directory, binary location, ~/.cargo/share/glossia/languages"))?;

    // Try to find payload file (supports subdirectories and recursive search)
    if let Some(payload_yaml) = find_language_file(language, &payload_filename) {
        return Ok(payload_yaml);
    }

    // Fallback: construct expected path for error message
    let expected_path = format!("{}/{}/{}", languages_dir, language, payload_filename);
    Err(format!("Wordlist file not found for language '{}'. Expected: {}\nOnly languages with a payload file are supported.",
                language, expected_path))
}

/// Find a language file (payload.yaml, cover.yaml, etc.) recursively
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn find_language_file(language: &str, filename: &str) -> Option<String> {
    let languages_dir = find_languages_dir()?;

    // First try exact path (supports subdirectories like "math/primes")
    let exact_path = format!("{}/{}/{}", languages_dir, language, filename);
    if std::path::Path::new(&exact_path).exists() {
        return Some(exact_path);
    }

    // If exact path doesn't exist, search recursively
    let languages_path = std::path::Path::new(&languages_dir);
    if let Ok(entries) = std::fs::read_dir(languages_path) {
        for entry in entries.flatten() {
            let path = entry.path();
            if path.is_dir() {
                if let Some(found) = find_language_file_recursive(&path, language, filename) {
                    return Some(found);
                }
            }
        }
    }

    None
}

/// Recursively search for a language file matching the language name
#[cfg(not(target_arch = "wasm32"))]
fn find_language_file_recursive(dir: &std::path::Path, language: &str, filename: &str) -> Option<String> {
    // Check if this directory matches the language name (last component)
    if let Some(dir_name) = dir.file_name().and_then(|n| n.to_str()) {
        // If language is just the directory name (e.g., "primes" matches "primes/")
        if dir_name == language {
            let file_path = dir.join(filename);
            if file_path.exists() {
                return Some(file_path.to_string_lossy().to_string());
            }
        }

        // If language contains slashes (e.g., "math/primes"), check if path ends with it
        if language.contains('/') {
            let lang_path = std::path::Path::new(language);
            if dir.ends_with(lang_path) {
                let file_path = dir.join(filename);
                if file_path.exists() {
                    return Some(file_path.to_string_lossy().to_string());
                }
            }
        }
    }

    // Recursively search subdirectories
    if let Ok(entries) = std::fs::read_dir(dir) {
        for entry in entries.flatten() {
            let path = entry.path();
            if path.is_dir() {
                if let Some(found) = find_language_file_recursive(&path, language, filename) {
                    return Some(found);
                }
            }
        }
    }

    None
}

// ═══════════════════════════════════════════════════════════════════════
// Dialect Detection
// ═══════════════════════════════════════════════════════════════════════

/// A scored candidate from dialect detection.
#[derive(Clone, Debug)]
pub struct DialectMatch {
    /// Language name (e.g., "english", "latin", "cs").
    pub language: String,
    /// Wordlist profile (e.g., "default", "bip39", "base58").
    pub wordlist: String,
    /// Available dialects for this language (e.g., ["body", "subject", "prose"]).
    pub dialects: Vec<String>,
    /// Number of input word tokens that matched this payload wordlist, counted
    /// **with multiplicity** (a payload word appearing 3 times counts as 3).
    pub hits: usize,
    /// Total number of input word tokens tested (with duplicates).
    pub total: usize,
    /// Hit rate: `hits / total` (0.0 to 1.0). Because both numerator and
    /// denominator count tokens with multiplicity, this is length-invariant —
    /// it does not decay as the input grows (see issue #26).
    pub hit_rate: f64,
    /// Size of the payload wordlist.
    pub wordlist_size: usize,
}

impl std::fmt::Display for DialectMatch {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
            "{}/{} — {}/{} hits ({:.1}%), wordlist size: {}",
            self.language,
            self.wordlist,
            self.hits,
            self.total,
            self.hit_rate * 100.0,
            self.wordlist_size,
        )
    }
}

/// An allowlist + threshold restricting which dialects [`detect_dialect_with`]
/// scans.
///
/// By default (`DialectFilter::default()` / `DialectFilter::new()`) every
/// language × wordlist combination is scanned with no hit-rate threshold —
/// identical to [`detect_dialect`]. Add constraints with the builder methods:
///
/// * [`min_hit_rate`](DialectFilter::min_hit_rate) — drop matches below a
///   `hits / total` ratio (footgun reduction; see issue #14).
/// * [`allow_language`](DialectFilter::allow_language) — restrict the scan to
///   whole languages.
/// * [`allow_wordlist`](DialectFilter::allow_wordlist) — restrict the scan to
///   specific `(language, wordlist)` pairs.
///
/// Allowlist semantics are a **union**: if any language or wordlist constraint
/// is set, a `(language, wordlist)` pair is scanned when it matches an allowed
/// whole-language **or** an allowed specific pair. If no allowlist constraint is
/// set, all pairs are scanned. Restricting the scan up front avoids the binary
/// searches — and, more importantly, any downstream `WordlistTree` cold-builds —
/// for excluded lists entirely.
///
/// # Example
///
/// ```no_run
/// use glossia::generator::{DialectFilter, detect_dialect_with};
///
/// // Only consider Latin's default list and English's bip39 list, and require
/// // a majority of input words to match.
/// let filter = DialectFilter::new()
///     .min_hit_rate(0.5)
///     .allow_wordlist("latin", "default")
///     .allow_wordlist("english", "bip39");
///
/// let words: Vec<String> = "abandon ability able".split_whitespace()
///     .map(|s| s.to_string()).collect();
/// let matches = detect_dialect_with(&words, &filter);
/// ```
#[derive(Clone, Debug, Default)]
pub struct DialectFilter {
    /// Minimum `hits / total` ratio (0.0..=1.0) a wordlist must reach to be kept.
    min_hit_rate: f64,
    /// If non-empty, whole languages that are always scanned.
    languages: HashSet<String>,
    /// If non-empty, specific `(language, wordlist)` pairs that are scanned.
    wordlists: HashSet<(String, String)>,
}

impl DialectFilter {
    /// Create an unconstrained filter (scans everything, no threshold).
    pub fn new() -> Self {
        Self::default()
    }

    /// Drop matches whose `hit_rate` is below `min_hit_rate` (0.0..=1.0).
    pub fn min_hit_rate(mut self, min_hit_rate: f64) -> Self {
        self.min_hit_rate = min_hit_rate;
        self
    }

    /// Allow an entire language (all of its wordlists).
    pub fn allow_language(mut self, language: impl Into<String>) -> Self {
        self.languages.insert(language.into());
        self
    }

    /// Allow several whole languages at once.
    pub fn allow_languages<I, S>(mut self, languages: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: Into<String>,
    {
        self.languages.extend(languages.into_iter().map(Into::into));
        self
    }

    /// Allow a specific `(language, wordlist)` pair.
    pub fn allow_wordlist(
        mut self,
        language: impl Into<String>,
        wordlist: impl Into<String>,
    ) -> Self {
        self.wordlists.insert((language.into(), wordlist.into()));
        self
    }

    /// Allow several specific `(language, wordlist)` pairs at once.
    pub fn allow_wordlists<I, L, W>(mut self, wordlists: I) -> Self
    where
        I: IntoIterator<Item = (L, W)>,
        L: Into<String>,
        W: Into<String>,
    {
        self.wordlists
            .extend(wordlists.into_iter().map(|(l, w)| (l.into(), w.into())));
        self
    }

    /// Whether any allowlist constraint (language or wordlist) has been set.
    fn has_allowlist(&self) -> bool {
        !self.languages.is_empty() || !self.wordlists.is_empty()
    }

    /// Whether the given `(language, wordlist)` pair should be scanned.
    fn allows(&self, language: &str, wordlist: &str) -> bool {
        if !self.has_allowlist() {
            return true;
        }
        if self.languages.contains(language) {
            return true;
        }
        // Avoid allocating a tuple of owned Strings just to probe the set.
        self.wordlists
            .iter()
            .any(|(l, w)| l == language && w == wordlist)
    }
}

/// Binary search for a word in a sorted, newline-delimited string.
///
/// The `sorted_text` parameter is a `&'static str` of words separated by `\n`,
/// sorted lexicographically (as produced by `build.rs` at compile time).
/// Returns `true` if `word` is found as an exact line match.
///
/// Complexity: O(log n) string comparisons where n = number of words.
fn binary_search_sorted_words(sorted_text: &str, word: &str) -> bool {
    let bytes = sorted_text.as_bytes();
    if bytes.is_empty() {
        return false;
    }

    let mut lo: usize = 0;
    let mut hi: usize = bytes.len();

    while lo < hi {
        let mid = lo + (hi - lo) / 2;

        // Find the start of the line containing byte position `mid`.
        let line_start = if mid == 0 || bytes[mid - 1] == b'\n' {
            mid
        } else {
            let mut pos = mid;
            while pos > lo && bytes[pos - 1] != b'\n' {
                pos -= 1;
            }
            pos
        };

        // Find the end of the line (next newline or end of string).
        let mut line_end = line_start;
        while line_end < bytes.len() && bytes[line_end] != b'\n' {
            line_end += 1;
        }

        let line = &sorted_text[line_start..line_end];

        match line.cmp(word) {
            std::cmp::Ordering::Equal => return true,
            std::cmp::Ordering::Less => lo = line_end + 1,
            std::cmp::Ordering::Greater => {
                if line_start == 0 {
                    return false;
                }
                hi = line_start;
            }
        }
    }
    false
}

/// Detect which dialect (language + wordlist) best matches the given input words.
///
/// Scans all available language × wordlist combinations and counts how many
/// of the input words appear in each payload wordlist. Results are sorted by
/// hit count (descending), then by hit rate.
///
/// # Arguments
///
/// * `input_words` — the words to test (typically extracted from encoded text
///   by splitting on whitespace and normalizing).
///
/// # Returns
///
/// A vector of `DialectMatch` entries sorted best-first. Only entries with
/// at least one hit are included.
///
/// # Footgun warning
///
/// This returns **every** wordlist with at least one hit, so a single
/// coincidentally-shared word can qualify a very large wordlist as a candidate.
/// Callers that build a `WordlistTree` per returned candidate (e.g. to attempt
/// a decode) can pay a large cold-build cost for lists that matched only
/// incidentally. Either threshold on `hit_rate` / `wordlist_size` before
/// building per-candidate trees, or use [`detect_dialect_filtered`] /
/// [`detect_dialect_with`] to drop low-hit-rate matches (and restrict the scan
/// to an allowlist of languages/wordlists) up front.
///
/// # Example
///
/// ```no_run
/// use glossia::generator::detect_dialect;
///
/// let words: Vec<String> = "abandon ability able about above".split_whitespace()
///     .map(|s| s.to_string()).collect();
/// let matches = detect_dialect(&words);
/// if let Some(best) = matches.first() {
///     println!("Detected: {}", best);
/// }
/// ```
pub fn detect_dialect(input_words: &[String]) -> Vec<DialectMatch> {
    detect_dialect_with(input_words, &DialectFilter::new())
}

/// Like [`detect_dialect`], but drops matches whose `hit_rate` is below
/// `min_hit_rate`.
///
/// This is the footgun-reduction variant: a `min_hit_rate` above `0.0` filters
/// out wordlists that matched only a handful of coincidental words, so callers
/// that build a `WordlistTree` per candidate don't pay the cold-build cost for
/// obviously-spurious large lists. For language/wordlist allowlisting as well,
/// use [`detect_dialect_with`].
///
/// # Arguments
///
/// * `input_words` — the words to test (typically extracted from encoded text
///   by splitting on whitespace and normalizing).
/// * `min_hit_rate` — minimum `hits / total` ratio (0.0 to 1.0) a wordlist must
///   reach to be included. `0.0` keeps every wordlist with at least one hit
///   (identical to [`detect_dialect`]); e.g. `0.5` keeps only wordlists matching
///   a majority of the input words.
///
/// # Returns
///
/// A vector of `DialectMatch` entries sorted best-first (most hits → highest
/// hit rate → smallest wordlist).
///
/// # Example
///
/// ```no_run
/// use glossia::generator::detect_dialect_filtered;
///
/// let words: Vec<String> = "abandon ability able about above".split_whitespace()
///     .map(|s| s.to_string()).collect();
/// // Only keep wordlists matching at least half the input words.
/// let matches = detect_dialect_filtered(&words, 0.5);
/// ```
pub fn detect_dialect_filtered(input_words: &[String], min_hit_rate: f64) -> Vec<DialectMatch> {
    detect_dialect_with(input_words, &DialectFilter::new().min_hit_rate(min_hit_rate))
}

/// Detect dialect, restricting the scan with a [`DialectFilter`].
///
/// This is the most general entry point: it backs both [`detect_dialect`] (with
/// an unconstrained filter) and [`detect_dialect_filtered`]. Use it to combine a
/// `min_hit_rate` threshold with an allowlist of whole languages and/or specific
/// `(language, wordlist)` pairs — pairs outside the allowlist are skipped before
/// any binary search, so excluded (and potentially large) lists cost nothing.
///
/// # Arguments
///
/// * `input_words` — the words to test (typically extracted from encoded text
///   by splitting on whitespace and normalizing).
/// * `filter` — the allowlist + hit-rate threshold; see [`DialectFilter`].
///
/// # Returns
///
/// A vector of `DialectMatch` entries sorted best-first (most hits → highest
/// hit rate → smallest wordlist).
///
/// # Example
///
/// ```no_run
/// use glossia::generator::{DialectFilter, detect_dialect_with};
///
/// let filter = DialectFilter::new()
///     .min_hit_rate(0.5)
///     .allow_language("latin")
///     .allow_wordlist("english", "bip39");
/// let words: Vec<String> = "abandon ability able".split_whitespace()
///     .map(|s| s.to_string()).collect();
/// let matches = detect_dialect_with(&words, &filter);
/// ```
pub fn detect_dialect_with(input_words: &[String], filter: &DialectFilter) -> Vec<DialectMatch> {
    use crate::grammar::DialectConfig;

    if input_words.is_empty() {
        return Vec::new();
    }

    // Normalize input words to lowercase, stripping trailing punctuation
    let normalized: Vec<String> = input_words
        .iter()
        .map(|w| w.trim_end_matches('.').trim_end_matches(',').to_lowercase())
        .filter(|w| !w.is_empty())
        .collect();

    let total = normalized.len();

    // Count occurrences of each distinct word. The binary searches run once per
    // *distinct* word (the keys), but `hit_rate` is scored with *multiplicity*
    // (see the loop below), so we keep the per-word counts here.
    let mut input_counts: HashMap<&str, usize> = HashMap::new();
    for w in &normalized {
        *input_counts.entry(w.as_str()).or_insert(0) += 1;
    }

    let mut results: Vec<DialectMatch> = Vec::new();

    let languages = get_available_languages();
    for &lang in languages {
        let wordlists = get_available_wordlists(lang);
        for wl_name in &wordlists {
            // Skip wordlists outside the allowlist before doing any work — this
            // avoids the binary searches and any downstream WordlistTree builds
            // for excluded (and potentially very large) lists.
            if !filter.allows(lang, wl_name) {
                continue;
            }

            // Every payload wordlist has a precomputed sorted index (built at compile time
            // alongside the disjointness and power-of-two checks). Binary search gives
            // O(log n) per word.
            let sorted_words = match language_index::get_payload_word_index(lang, wl_name) {
                Some(w) => w,
                None => continue,
            };

            // Count matched words *with multiplicity*: every input token that is
            // a payload word counts, not just the distinct ones. Scoring distinct
            // matches over a with-duplicates total (the old behavior) made
            // `hit_rate` decay as ~1/length, so long-but-valid encoded bodies sank
            // below downstream thresholds purely as a function of length (issue
            // #26). An occurrence-over-total ratio is length- and entropy-
            // invariant, which is what makes the issue #14 `min_hit_rate` filter a
            // trustworthy confidence signal rather than a length-sensitive one.
            let mut hits = 0usize;
            for (word, &count) in &input_counts {
                if binary_search_sorted_words(sorted_words, word) {
                    hits += count;
                }
            }

            if hits > 0 {
                let hit_rate = hits as f64 / total as f64;
                if hit_rate < filter.min_hit_rate {
                    continue;
                }

                let wordlist_size = language_index::get_payload_word_count(lang, wl_name);
                let dialects = DialectConfig::available_dialects(lang);

                results.push(DialectMatch {
                    language: lang.to_string(),
                    wordlist: wl_name.clone(),
                    dialects,
                    hits,
                    total,
                    hit_rate,
                    wordlist_size,
                });
            }
        }
    }

    // Sort: most hits first, then highest hit rate, then smallest wordlist (more specific)
    results.sort_by(|a, b| {
        b.hits.cmp(&a.hits)
            .then_with(|| b.hit_rate.partial_cmp(&a.hit_rate).unwrap_or(std::cmp::Ordering::Equal))
            .then_with(|| a.wordlist_size.cmp(&b.wordlist_size))
    });

    results
}

/// Detect dialect and return the best match, or None if no payload words matched.
///
/// This is a convenience wrapper around [`detect_dialect`].
pub fn detect_dialect_best(input_words: &[String]) -> Option<DialectMatch> {
    detect_dialect(input_words).into_iter().next()
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::collections::HashSet;
    use crate::types::Pos;

    #[test]
    fn test_cover_yaml_loads_refinements() {
        let wordlist_set: HashSet<String> = HashSet::new();
        let (by_pos, refined_cover) = load_cover_words_by_pos(&wordlist_set, "english");

        let def_dets = refined_cover.get(&(Pos::Det, "def".to_string()));
        assert!(def_dets.is_some(), "Should have refined_cover entry for (Det, def)");
        let def_dets = def_dets.unwrap();
        assert!(def_dets.contains(&"the".to_string()), "Det[def] should contain 'the'");
        assert!(def_dets.contains(&"its".to_string()), "Det[def] should contain 'its'");
        assert!(def_dets.contains(&"our".to_string()), "Det[def] should contain 'our'");

        let indef_dets = refined_cover.get(&(Pos::Det, "indef".to_string()));
        assert!(indef_dets.is_some(), "Should have refined_cover entry for (Det, indef)");
        let indef_dets = indef_dets.unwrap();
        assert!(indef_dets.contains(&"a".to_string()), "Det[indef] should contain 'a'");
        assert!(indef_dets.contains(&"an".to_string()), "Det[indef] should contain 'an'");

        let sg_cops = refined_cover.get(&(Pos::Cop, "sg".to_string()));
        assert!(sg_cops.is_some(), "Should have refined_cover entry for (Cop, sg)");
        let sg_cops = sg_cops.unwrap();
        assert!(sg_cops.contains(&"is".to_string()), "Cop[sg] should contain 'is'");

        let pl_cops = refined_cover.get(&(Pos::Cop, "pl".to_string()));
        assert!(pl_cops.is_some(), "Should have refined_cover entry for (Cop, pl)");
        let pl_cops = pl_cops.unwrap();
        assert!(pl_cops.contains(&"are".to_string()), "Cop[pl] should contain 'are'");

        let quant_dets = refined_cover.get(&(Pos::Det, "quant".to_string()));
        assert!(quant_dets.is_some(), "Should have refined_cover entry for (Det, quant)");
        let quant_dets = quant_dets.unwrap();
        assert!(quant_dets.contains(&"each".to_string()), "Det[quant] should contain 'each'");
        assert!(quant_dets.contains(&"every".to_string()), "Det[quant] should contain 'every'");
        assert!(quant_dets.contains(&"some".to_string()), "Det[quant] should contain 'some'");

        let all_dets = by_pos.get(&Pos::Det);
        assert!(all_dets.is_some(), "by_pos should have Det");
        let all_dets = all_dets.unwrap();
        assert!(all_dets.contains(&"the".to_string()), "by_pos[Det] should contain 'the'");
        assert!(all_dets.contains(&"a".to_string()), "by_pos[Det] should contain 'a'");
    }

    #[test]
    fn test_is_and_are_load_as_cop() {
        let wordlist_set: HashSet<String> = HashSet::new();
        let (by_pos, _) = load_cover_words_by_pos(&wordlist_set, "english");

        let cop_words = by_pos.get(&Pos::Cop);
        assert!(cop_words.is_some(), "Should have Cop words");
        let cop_words = cop_words.unwrap();
        assert!(cop_words.contains(&"is".to_string()), "'is' should be in Cop");
        assert!(cop_words.contains(&"are".to_string()), "'are' should be in Cop");

        if let Some(v_words) = by_pos.get(&Pos::V) {
            assert!(!v_words.contains(&"is".to_string()), "'is' should NOT be in V");
            assert!(!v_words.contains(&"are".to_string()), "'are' should NOT be in V");
        }
    }

    #[test]
    fn test_words_without_refinement_not_in_refined_cover() {
        let wordlist_set: HashSet<String> = HashSet::new();
        let (by_pos, refined_cover) = load_cover_words_by_pos(&wordlist_set, "english");

        let adj_words = by_pos.get(&Pos::Adj);
        assert!(adj_words.is_some(), "Should have Adj words");
        let adj_words = adj_words.unwrap();
        assert!(adj_words.contains(&"bad".to_string()), "'bad' should be in by_pos[Adj]");

        let in_refined = refined_cover.iter().any(|(_, words)| {
            words.contains(&"bad".to_string())
        });
        assert!(!in_refined, "'bad' should NOT appear in refined_cover (no refinement tag)");
    }

    #[test]
    fn test_cover_words_exclude_payload_words() {
        let mut wordlist_set: HashSet<String> = HashSet::new();
        wordlist_set.insert("aid".to_string());

        let (by_pos, _) = load_cover_words_by_pos(&wordlist_set, "english");

        for (_, words) in &by_pos {
            assert!(!words.contains(&"aid".to_string()),
                "'aid' should be excluded from cover words when it is in the wordlist set");
        }
    }

    #[test]
    fn test_load_payload_words_english() {
        let words = load_payload_words("english").expect("Should load English payload words");
        assert!(!words.is_empty(), "English payload words should not be empty");
        assert!(words.contains(&"abandon".to_string()), "Should contain 'abandon'");
        let mut sorted = words.clone();
        sorted.sort();
        assert_eq!(words, sorted, "Payload words should be sorted");
    }

    #[test]
    fn test_build_pos_mapping_english() {
        let mapping = build_pos_mapping("english").expect("Should build English POS mapping");
        assert!(!mapping.is_empty(), "POS mapping should not be empty");

        let abandon_pos = mapping.get("abandon");
        assert!(abandon_pos.is_some(), "'abandon' should have POS tags");
        let abandon_pos = abandon_pos.unwrap();
        assert!(!abandon_pos.is_empty(), "'abandon' should have at least one POS tag");
    }

    #[test]
    fn test_pos_from_str_all_variants() {
        for pos in Pos::ALL {
            let s = pos.as_str();
            assert_eq!(Pos::from_str(s), Some(*pos), "round-trip failed for {:?}", pos);
        }
        assert_eq!(Pos::from_str("Unknown"), None);
    }

    // ── Dialect detection tests ──────────────────────────────────────

    #[test]
    fn test_detect_dialect_bip39_words() {
        // These are all BIP39 words → should detect english/default (or english/bip39)
        let words: Vec<String> = "abandon ability able about above"
            .split_whitespace()
            .map(|s| s.to_string())
            .collect();
        let matches = detect_dialect(&words);
        assert!(!matches.is_empty(), "Should detect at least one dialect for BIP39 words");

        let best = &matches[0];
        assert_eq!(best.language, "english", "Best match should be English");
        assert_eq!(best.hits, 5, "All 5 words should be hits");
        assert!((best.hit_rate - 1.0).abs() < 0.001, "Hit rate should be 1.0");
    }

    #[test]
    fn test_detect_dialect_hit_rate_length_invariant() {
        // Regression for issue #26: `hit_rate` must not decay with input length.
        // The old metric (distinct matches ÷ total-with-duplicates) fell as
        // ~1/length; the occurrence-based metric stays flat. Repeating an
        // all-payload-word input lengthens it without diluting the rate.
        let base = ["abandon", "ability", "able"];
        let short: Vec<String> = base.iter().map(|s| s.to_string()).collect();
        let long: Vec<String> = base
            .iter()
            .cycle()
            .take(base.len() * 20)
            .map(|s| s.to_string())
            .collect();

        let short_best = detect_dialect_best(&short).expect("short input should detect");
        let long_best = detect_dialect_best(&long).expect("long input should detect");

        // Every token is a payload word, so the rate is 1.0 regardless of length.
        assert!(
            (short_best.hit_rate - 1.0).abs() < 1e-9,
            "short hit_rate should be 1.0, got {}",
            short_best.hit_rate
        );
        assert!(
            (long_best.hit_rate - 1.0).abs() < 1e-9,
            "long hit_rate should be 1.0, got {} — metric decayed with length",
            long_best.hit_rate
        );
        // hits counts with multiplicity: 60 tokens, all matching.
        assert_eq!(long_best.hits, base.len() * 20);
        assert_eq!(long_best.total, base.len() * 20);
    }

    #[test]
    fn test_detect_dialect_empty_input() {
        let words: Vec<String> = Vec::new();
        let matches = detect_dialect(&words);
        assert!(matches.is_empty(), "Empty input should return no matches");
    }

    #[test]
    fn test_detect_dialect_no_matches() {
        // These words shouldn't be in any payload wordlist
        let words: Vec<String> = "xyzzyplugh fnord"
            .split_whitespace()
            .map(|s| s.to_string())
            .collect();
        let matches = detect_dialect(&words);
        assert!(matches.is_empty(), "Nonsense words should return no matches");
    }

    #[test]
    fn test_detect_dialect_filtered_drops_low_hit_rate() {
        // "abandon", "ability", "able" are BIP39 payload words; "xyzzyplugh" is not.
        // With a high min_hit_rate, wordlists that match only a coincidental
        // fraction of the input should be dropped.
        let words: Vec<String> = "abandon ability able xyzzyplugh"
            .split_whitespace()
            .map(|s| s.to_string())
            .collect();

        // min_hit_rate = 0.0 is identical to detect_dialect: keep every hit.
        let unfiltered = detect_dialect_filtered(&words, 0.0);
        assert_eq!(unfiltered.len(), detect_dialect(&words).len());

        // A 0.5 threshold keeps the strong English match (3/4 = 0.75) but every
        // returned candidate must clear the bar.
        let filtered = detect_dialect_filtered(&words, 0.5);
        assert!(!filtered.is_empty(), "Strong match should survive filtering");
        for m in &filtered {
            assert!(
                m.hit_rate >= 0.5,
                "filtered result {} has hit_rate {} below threshold",
                m, m.hit_rate
            );
        }

        // An impossible threshold drops everything.
        let none = detect_dialect_filtered(&words, 1.01);
        assert!(none.is_empty(), "hit_rate can't exceed 1.0, so all should be dropped");
    }

    #[test]
    fn test_detect_dialect_with_language_allowlist() {
        // BIP39 words live in english; restricting to a non-english language
        // should yield no english matches.
        let words: Vec<String> = "abandon ability able about above"
            .split_whitespace()
            .map(|s| s.to_string())
            .collect();

        // Unconstrained filter is identical to detect_dialect.
        let all = detect_dialect_with(&words, &DialectFilter::new());
        assert_eq!(all.len(), detect_dialect(&words).len());
        assert!(all.iter().any(|m| m.language == "english"));

        // Allow only english: every returned match must be english.
        let only_english = detect_dialect_with(&words, &DialectFilter::new().allow_language("english"));
        assert!(!only_english.is_empty(), "english should still match BIP39 words");
        assert!(
            only_english.iter().all(|m| m.language == "english"),
            "language allowlist must exclude non-english matches"
        );
    }

    #[test]
    fn test_detect_dialect_with_wordlist_allowlist_union() {
        // BIP39 words: english has multiple wordlists. Restrict to a single
        // (english, bip39) pair and confirm nothing else is returned.
        let words: Vec<String> = "abandon ability able about above"
            .split_whitespace()
            .map(|s| s.to_string())
            .collect();

        let filter = DialectFilter::new().allow_wordlist("english", "bip39");
        let matches = detect_dialect_with(&words, &filter);
        assert!(!matches.is_empty(), "english/bip39 should match BIP39 words");
        assert!(
            matches.iter().all(|m| m.language == "english" && m.wordlist == "bip39"),
            "wordlist allowlist must restrict to the exact (language, wordlist) pair"
        );

        // Union semantics: allowing a whole language OR a specific pair both pass.
        let union = detect_dialect_with(
            &words,
            &DialectFilter::new()
                .allow_language("english")
                .allow_wordlist("latin", "default"),
        );
        assert!(
            union.iter().all(|m| m.language == "english" || (m.language == "latin" && m.wordlist == "default")),
            "union allowlist should admit english (any wordlist) and latin/default only"
        );
    }

    #[test]
    fn test_detect_dialect_mixed_payload_and_cover() {
        // Mix of payload words and cover/non-payload words
        // "the" is a cover word, "abandon" and "zoo" are BIP39 payload words
        let words: Vec<String> = "the abandon is zoo"
            .split_whitespace()
            .map(|s| s.to_string())
            .collect();
        let matches = detect_dialect(&words);
        assert!(!matches.is_empty(), "Should detect dialect from mixed words");

        let best = &matches[0];
        assert_eq!(best.language, "english");
        // Should have 2 hits (abandon, zoo) — "the" and "is" are cover words
        assert!(best.hits >= 2, "Should have at least 2 payload hits, got {}", best.hits);
    }

    #[test]
    fn test_detect_dialect_strips_punctuation() {
        // Words with trailing punctuation should still match
        let words: Vec<String> = "abandon. ability, able"
            .split_whitespace()
            .map(|s| s.to_string())
            .collect();
        let matches = detect_dialect(&words);
        assert!(!matches.is_empty(), "Should detect dialect despite punctuation");

        let best = &matches[0];
        assert_eq!(best.hits, 3, "All 3 words should match after stripping punctuation");
    }

    #[test]
    fn test_detect_dialect_best_convenience() {
        let words: Vec<String> = "abandon ability"
            .split_whitespace()
            .map(|s| s.to_string())
            .collect();
        let best = detect_dialect_best(&words);
        assert!(best.is_some(), "Should detect best dialect");
        assert_eq!(best.unwrap().language, "english");
    }

    #[test]
    fn test_detect_dialect_includes_dialects() {
        let words: Vec<String> = "abandon ability able"
            .split_whitespace()
            .map(|s| s.to_string())
            .collect();
        let matches = detect_dialect(&words);
        let best = &matches[0];
        // English should have body, subject, prose dialects
        assert!(best.dialects.contains(&"body".to_string()), "Should include body dialect");
        assert!(best.dialects.contains(&"subject".to_string()), "Should include subject dialect");
    }

    #[test]
    fn test_binary_search_sorted_words() {
        let text = "apple\nbanana\ncherry\ndate\nelderberry";

        // Found cases
        assert!(binary_search_sorted_words(text, "apple"), "should find first word");
        assert!(binary_search_sorted_words(text, "cherry"), "should find middle word");
        assert!(binary_search_sorted_words(text, "elderberry"), "should find last word");
        assert!(binary_search_sorted_words(text, "banana"), "should find second word");
        assert!(binary_search_sorted_words(text, "date"), "should find fourth word");

        // Not found cases
        assert!(!binary_search_sorted_words(text, "aaa"), "should not find word before first");
        assert!(!binary_search_sorted_words(text, "zzz"), "should not find word after last");
        assert!(!binary_search_sorted_words(text, "car"), "should not find word between entries");
        assert!(!binary_search_sorted_words(text, ""), "should not find empty string");

        // Edge cases
        assert!(!binary_search_sorted_words("", "apple"), "empty text should find nothing");
        assert!(binary_search_sorted_words("solo", "solo"), "single word should be found");
        assert!(!binary_search_sorted_words("solo", "other"), "single word should reject others");
    }

    #[test]
    fn test_detect_dialect_uses_precomputed_index() {
        // Verify exact wordlist_size from precomputed index (not estimated from YAML length)
        let words: Vec<String> = "abandon ability able about above"
            .split_whitespace()
            .map(|s| s.to_string())
            .collect();
        let matches = detect_dialect(&words);
        let best = &matches[0];
        assert_eq!(best.language, "english");
        assert_eq!(best.wordlist, "bip39");
        // BIP39 has exactly 2048 words — precomputed, not estimated
        assert_eq!(best.wordlist_size, 2048, "wordlist_size should be exact (2048), not an estimate");
    }
}