euhadra 0.3.0

use crate::types::{Language, Span};
use async_trait::async_trait;

// ---------------------------------------------------------------------------
// TextFilter trait
// ---------------------------------------------------------------------------

/// A lightweight text transformation applied between ASR and LLM refinement.
///
/// Filters are cheaper than full LLM calls and handle mechanical
/// transformations: filler removal, profanity filtering, normalization, etc.
/// Multiple filters can be chained.
#[async_trait]
pub trait TextFilter: Send + Sync {
    /// Transform text, returning the filtered version and a list of removed
    /// segments (for diagnostics / undo).
    async fn filter(&self, text: &str) -> Result<FilterResult, FilterError>;
}

/// What a [`TextFilter`] did to the text.
///
/// `#[non_exhaustive]`: callers read this, and per-segment positions are
/// a plausible future addition.
#[derive(Debug, Clone, Default)]
#[non_exhaustive]
pub struct FilterResult {
    /// The text after filtering.
    pub text: String,
    /// The segments that were removed, in text order — for diagnostics
    /// and for undo.
    pub removed: Vec<String>,
}

/// Why a [`TextFilter`] could not filter its input.
#[derive(Debug, Clone, thiserror::Error)]
#[non_exhaustive]
pub enum FilterError {
    /// A resource the filter depends on could not be loaded — an
    /// embedding model, a dictionary file.
    #[error("filter resource unavailable: {0}")]
    Unavailable(String),

    /// The filter ran but could not complete.
    #[error("filter failed: {0}")]
    Failed(String),
}

// ---------------------------------------------------------------------------
// SimpleFillerFilter — tiered, position-aware, zero-dependency
// ---------------------------------------------------------------------------

/// The filler filter for a given language.
///
/// This is the entry point to use unless you need a specific filter's
/// language-specific configuration. Constructing a concrete filter directly
/// means pairing it with a language yourself, and the two are not
/// interchangeable: [`SimpleFillerFilter`] splits on whitespace, which
/// Japanese and Chinese text does not have. A whole utterance then arrives
/// as a single token, and if it opens with a filler the entire utterance is
/// removed — an empty transcript, with no error. `for_language` makes that
/// pairing unavailable to get wrong.
///
/// ```
/// use euhadra::prelude::*;
///
/// # async fn f() -> Result<(), FilterError> {
/// let filter = FillerFilter::for_language(Language::Japanese);
/// let result = filter.filter("えーと、今日は会議があります。").await?;
/// assert_eq!(result.text, "今日は会議があります。");
/// # Ok(())
/// # }
/// ```
#[derive(Debug, Clone)]
#[non_exhaustive]
pub enum FillerFilter {
    /// Whitespace-tokenised filtering — English, Spanish, Korean.
    Word(SimpleFillerFilter),
    /// Comma-segmented filtering for Japanese (、).
    Japanese(JapaneseFillerFilter),
    /// Comma-segmented filtering for Chinese (，).
    Chinese(ChineseFillerFilter),
    /// Spanish-specific filtering, including multi-word discourse markers.
    Spanish(SpanishFillerFilter),
}

impl FillerFilter {
    /// The filler filter matching `language`'s script and segmentation.
    ///
    /// Total over [`Language`] — every supported language has a filter, so
    /// there is no failure case and no silent fallback to a wrong one.
    pub fn for_language(language: Language) -> Self {
        match language {
            Language::English => Self::Word(SimpleFillerFilter::english()),
            Language::Korean => Self::Word(SimpleFillerFilter::korean()),
            Language::Japanese => Self::Japanese(JapaneseFillerFilter::new()),
            Language::Chinese => Self::Chinese(ChineseFillerFilter::new()),
            Language::Spanish => Self::Spanish(SpanishFillerFilter::new()),
        }
    }

    /// The codepoint span of every segment this filter would remove.
    ///
    /// Spans are what the L3 direct-F1 evaluator scores against the gold
    /// annotations, and what a caller needs to highlight or undo a removal.
    pub fn detect_spans(&self, text: &str) -> Vec<Span> {
        match self {
            Self::Word(f) => f.detect_spans(text),
            Self::Japanese(f) => f.detect_spans(text),
            Self::Chinese(f) => f.detect_spans(text),
            Self::Spanish(f) => f.detect_spans(text),
        }
    }
}

#[async_trait]
impl TextFilter for FillerFilter {
    async fn filter(&self, text: &str) -> Result<FilterResult, FilterError> {
        match self {
            Self::Word(f) => f.filter(text).await,
            Self::Japanese(f) => f.filter(text).await,
            Self::Chinese(f) => f.filter(text).await,
            Self::Spanish(f) => f.filter(text).await,
        }
    }
}

/// Filler removal using tiered word matching and position heuristics.
///
/// Fillers are split into two tiers:
/// - **Pure fillers** (um, uh, er, ah) — removed unconditionally.
/// - **Contextual fillers** (so, well, basically, actually) — removed only at
///   sentence-initial position, since they serve as discourse markers there
///   but act as real content words mid-sentence ("it went well", "do so").
///
/// Multi-word fillers ("you know", "I mean") are matched as bigrams.
#[derive(Debug, Clone)]
pub struct SimpleFillerFilter {
    /// Always removed regardless of position.
    pure_fillers: Vec<String>,
    /// Removed only when sentence-initial (first word, or after punctuation).
    contextual_fillers: Vec<String>,
    /// Multi-word fillers matched as consecutive word pairs.
    multi_fillers: Vec<Vec<String>>,
}

impl SimpleFillerFilter {
    pub fn english() -> Self {
        Self {
            pure_fillers: vec!["um", "uh", "uhm", "umm", "hmm", "er", "ah", "eh"]
                .into_iter()
                .map(String::from)
                .collect(),

            contextual_fillers: vec!["so", "well", "basically", "actually", "literally", "right"]
                .into_iter()
                .map(String::from)
                .collect(),

            multi_fillers: vec![
                vec!["you".into(), "know".into()],
                vec!["i".into(), "mean".into()],
                vec!["you".into(), "see".into()],
                vec!["sort".into(), "of".into()],
                vec!["kind".into(), "of".into()],
                // Carried over from the retired Python `filler_filter.py`
                // lexicon, which had these two and the Rust filter did not.
                vec!["or".into(), "something".into()],
                vec!["or".into(), "whatever".into()],
            ],
        }
    }

    pub fn with_pure_fillers(mut self, fillers: Vec<String>) -> Self {
        self.pure_fillers = fillers;
        self
    }

    pub fn with_contextual_fillers(mut self, fillers: Vec<String>) -> Self {
        self.contextual_fillers = fillers;
        self
    }

    /// Korean filler filter.
    ///
    /// Korean is whitespace-segmented at the eojeol level — same
    /// tokenisation strategy as English / Spanish, not the
    /// comma-segmentation used for Japanese / Chinese. Pure-filler
    /// words (음 / 어 / 아 / 으) are removed at any position;
    /// contextual fillers (그 / 저 / 그러니까 / 막 / 약간 / 뭐 /
    /// 뭐랄까 / 저기 / 글쎄 / 어쨌든) only get stripped at
    /// sentence-initial positions because every one of them doubles
    /// as a content word mid-utterance (그 = "that" demonstrative,
    /// 저 = "I / that", 막 = "just (intensifier)" / "block (verb)",
    /// 뭐 = "what", etc.).
    pub fn korean() -> Self {
        Self {
            pure_fillers: vec![
                // Hesitation vowels — `음` / `어` / `아` / `으` are
                // the canonical lone-vowel hesitations in spoken
                // Korean. `엄` and `응` are softer variants that
                // appear in some ASR transcripts.
                "음", "어", "아", "으", "엄", "응",
            ]
            .into_iter()
            .map(String::from)
            .collect(),

            contextual_fillers: vec![
                // Discourse markers — filler at sentence start, content
                // word otherwise. `그` and `저` are the two most
                // ambiguous; the rest read as fillers nearly any time
                // they appear sentence-initial.
                "그",
                "저",
                "막",
                "약간",
                "뭐",
                "저기",
                "글쎄",
                "그러니까",
                "그니까",
                "그래서",
                "어쨌든",
                "그게",
                "뭐랄까",
            ]
            .into_iter()
            .map(String::from)
            .collect(),

            multi_fillers: vec![
                // CTC commonly emits these as two separated eojeol
                // even when they pronounce as a single hesitation
                // unit.
                vec!["음".into(), "그러니까".into()],
                vec!["그게".into(), "그러니까".into()],
                vec!["뭐랄까".into(), "그".into()],
            ],
        }
    }

    /// Returns true if position `i` is sentence-initial: either the first word,
    /// or preceded only by removed words / punctuation.
    fn is_sentence_initial(words: &[&str], idx: usize, removed_indices: &[bool]) -> bool {
        if idx == 0 {
            return true;
        }
        // Walk backwards — if every preceding word was removed or ends with
        // sentence-ending punctuation, this position is sentence-initial.
        for j in (0..idx).rev() {
            if removed_indices[j] {
                continue;
            }
            // Previous kept word ends with sentence punctuation → we're at a boundary
            let prev = words[j];
            if prev.ends_with('.')
                || prev.ends_with('!')
                || prev.ends_with('?')
                || prev.ends_with(',')
            {
                return true;
            }
            return false;
        }
        true
    }

    /// Run the 3-pass filler detector and return the codepoint span
    /// of every removed segment. Mirrors the Spanish filter's
    /// `detect_spans` so the L3 direct-F1 evaluator can compare
    /// emitted spans against the gold annotations in
    /// `tests/evaluation/annotations/{en,ko}_filler.jsonl`.
    pub fn detect_spans(&self, text: &str) -> Vec<Span> {
        let chars: Vec<char> = text.chars().collect();
        let n_chars = chars.len();

        // Tokenise on whitespace; codepoint-index spans match the
        // schema used by `Span` everywhere else in the F1 module.
        struct Tok {
            cp_start: usize,
            cp_end: usize,
            clean_lower: String,
            ends_with_terminal: bool,
        }
        let mut tokens: Vec<Tok> = Vec::new();
        let mut i = 0;
        while i < n_chars {
            while i < n_chars && chars[i].is_whitespace() {
                i += 1;
            }
            if i >= n_chars {
                break;
            }
            let start = i;
            while i < n_chars && !chars[i].is_whitespace() {
                i += 1;
            }
            let surface: String = chars[start..i].iter().collect();
            let lower = surface.to_lowercase();
            let clean: String = lower.chars().take_while(|c| c.is_alphanumeric()).collect();
            let ends_with_terminal = surface.ends_with('.')
                || surface.ends_with('!')
                || surface.ends_with('?')
                || surface.ends_with(',');
            tokens.push(Tok {
                cp_start: start,
                cp_end: i,
                clean_lower: clean,
                ends_with_terminal,
            });
        }

        let n = tokens.len();
        let mut removed = vec![false; n];
        let mut detections: Vec<(usize, usize)> = Vec::new();

        // Pass 1: 2-token multi-word fillers ("you know", "i mean", …)
        let mut k = 0;
        'outer: while k + 1 < n {
            for mf in &self.multi_fillers {
                if mf.len() == 2
                    && !removed[k]
                    && !removed[k + 1]
                    && tokens[k].clean_lower == mf[0]
                    && tokens[k + 1].clean_lower == mf[1]
                {
                    removed[k] = true;
                    removed[k + 1] = true;
                    detections.push((k, k + 1));
                    k += 2;
                    continue 'outer;
                }
            }
            k += 1;
        }

        // Pass 2: pure fillers (any position)
        for j in 0..n {
            if removed[j] {
                continue;
            }
            if self.pure_fillers.contains(&tokens[j].clean_lower) {
                removed[j] = true;
                detections.push((j, j));
            }
        }

        // Pass 3: contextual fillers — sentence-initial only.
        // Inline the sentence-initial check here so it can read the
        // token vector (the original `is_sentence_initial` works on
        // `&[&str]` words). Walk backwards; the first non-removed
        // predecessor decides.
        let is_initial = |idx: usize, rem: &[bool]| -> bool {
            if idx == 0 {
                return true;
            }
            for jj in (0..idx).rev() {
                if rem[jj] {
                    continue;
                }
                return tokens[jj].ends_with_terminal;
            }
            true
        };
        for j in 0..n {
            if removed[j] {
                continue;
            }
            if self.contextual_fillers.contains(&tokens[j].clean_lower) && is_initial(j, &removed) {
                removed[j] = true;
                detections.push((j, j));
            }
        }

        let mut spans: Vec<Span> = detections
            .iter()
            .map(|(s, e)| crate::types::Span {
                start: tokens[*s].cp_start,
                end: tokens[*e].cp_end,
            })
            .collect();
        spans.sort_by_key(|s| s.start);
        spans
    }
}

#[async_trait]
impl TextFilter for SimpleFillerFilter {
    async fn filter(&self, text: &str) -> Result<FilterResult, FilterError> {
        let words: Vec<&str> = text.split_whitespace().collect();
        let n = words.len();
        let mut removed_indices = vec![false; n];
        let mut removed_labels: Vec<String> = Vec::new();

        // Pass 1: remove multi-word fillers
        let mut i = 0;
        'outer: while i + 1 < n {
            for mf in &self.multi_fillers {
                if mf.len() == 2 && !removed_indices[i] && !removed_indices[i + 1] {
                    let w0 = words[i].to_lowercase();
                    let w1 = words[i + 1].to_lowercase();
                    let clean0: String = w0.chars().take_while(|c| c.is_alphanumeric()).collect();
                    let clean1: String = w1.chars().take_while(|c| c.is_alphanumeric()).collect();
                    if clean0 == mf[0] && clean1 == mf[1] {
                        removed_indices[i] = true;
                        removed_indices[i + 1] = true;
                        removed_labels.push(format!("{} {}", words[i], words[i + 1]));
                        i += 2;
                        // Skip the trailing `i += 1` and re-evaluate the
                        // while bound, otherwise the next iteration of
                        // `for mf` would index past the end when `i` has
                        // just been advanced to `n`.
                        continue 'outer;
                    }
                }
            }
            i += 1;
        }

        // Pass 2: remove pure fillers (unconditional)
        for i in 0..n {
            if removed_indices[i] {
                continue;
            }
            let lower = words[i].to_lowercase();
            let clean: String = lower.chars().take_while(|c| c.is_alphanumeric()).collect();
            if self.pure_fillers.contains(&clean) {
                removed_indices[i] = true;
                removed_labels.push(words[i].to_string());
            }
        }

        // Pass 3: remove contextual fillers only at sentence-initial position
        for i in 0..n {
            if removed_indices[i] {
                continue;
            }
            let lower = words[i].to_lowercase();
            let clean: String = lower.chars().take_while(|c| c.is_alphanumeric()).collect();
            if self.contextual_fillers.contains(&clean)
                && Self::is_sentence_initial(&words, i, &removed_indices)
            {
                removed_indices[i] = true;
                removed_labels.push(words[i].to_string());
            }
        }

        let kept: Vec<&str> = words
            .iter()
            .enumerate()
            .filter(|(i, _)| !removed_indices[*i])
            .map(|(_, w)| *w)
            .collect();

        Ok(FilterResult {
            text: kept.join(" "),
            removed: removed_labels,
        })
    }
}

// ---------------------------------------------------------------------------
// JapaneseFillerFilter — comma-segment based filler removal for Japanese
// ---------------------------------------------------------------------------

/// Filler removal for Japanese text from ASR output.
///
/// Japanese ASR output uses 、 (ideographic comma) as a natural delimiter.
/// Fillers appear as short segments between commas: "えーと、" "あの、" "まあ、".
/// This filter splits on 、, checks each segment against filler lexicons,
/// and removes matching segments.
///
/// Also handles ASR artifacts where whisper converts Japanese fillers into
/// kanji (e.g. えーと → 映像, えー → 映映).
#[derive(Debug, Clone)]
pub struct JapaneseFillerFilter {
    /// Always removed regardless of position.
    pure_fillers: Vec<String>,
    /// Removed only at sentence-initial position.
    contextual_fillers: Vec<String>,
}

impl JapaneseFillerFilter {
    pub fn new() -> Self {
        Self {
            pure_fillers: vec![
                "えーと",
                "えっと",
                "えー",
                "あー",
                "うーん",
                "うん",
                "ああ",
                "ええ",
                // Common ASR misrecognitions
                "映像",
                "映映",
            ]
            .into_iter()
            .map(String::from)
            .collect(),

            contextual_fillers: vec!["あの", "まあ", "その", "なんか", "ほら", "やっぱり"]
                .into_iter()
                .map(String::from)
                .collect(),
        }
    }
}

impl Default for JapaneseFillerFilter {
    fn default() -> Self {
        Self::new()
    }
}

#[async_trait]
impl TextFilter for JapaneseFillerFilter {
    async fn filter(&self, text: &str) -> Result<FilterResult, FilterError> {
        // Split by Japanese comma (、) keeping track of segments
        let segments: Vec<&str> = text.split('、').collect();
        let n = segments.len();
        let mut removed_indices = vec![false; n];
        let mut removed_labels: Vec<String> = Vec::new();

        // Pass 1: pure fillers — remove unconditionally
        for i in 0..n {
            let trimmed = segments[i].trim();
            if trimmed.is_empty() {
                continue;
            }
            if self.pure_fillers.iter().any(|f| trimmed == f) {
                removed_indices[i] = true;
                removed_labels.push(trimmed.to_string());
            }
        }

        // Pass 2: contextual fillers — remove only at sentence-initial position
        for i in 0..n {
            if removed_indices[i] {
                continue;
            }
            let trimmed = segments[i].trim();
            if trimmed.is_empty() {
                continue;
            }
            // Check if this is sentence-initial
            let is_initial = (0..i).all(|j| removed_indices[j] || segments[j].trim().is_empty());
            // Also check after sentence-ending punctuation (。)
            let after_period =
                i > 0 && !removed_indices[i - 1] && segments[i - 1].trim().ends_with('。');

            if self.contextual_fillers.iter().any(|f| trimmed == f) && (is_initial || after_period)
            {
                removed_indices[i] = true;
                removed_labels.push(trimmed.to_string());
            }
        }

        // Pass 3: standalone contextual fillers — Japanese-specific heuristic.
        //
        // In Japanese ASR output, fillers appear as independent comma-delimited
        // segments: "あの、お立ち合いの中に" → segments ["あの", "お立ち合いの中に"].
        //
        // A contextual filler word that IS the entire segment (not part of a
        // longer phrase) is almost certainly a filler, not a content word.
        // Compare: "あの" (standalone → filler) vs "あの人が来た" (part of
        // phrase → demonstrative "that person").
        for i in 0..n {
            if removed_indices[i] {
                continue;
            }
            let trimmed = segments[i].trim();
            if trimmed.is_empty() {
                continue;
            }
            // If the entire segment is exactly a contextual filler word,
            // it's a standalone filler — remove it.
            if self.contextual_fillers.iter().any(|f| trimmed == f) {
                removed_indices[i] = true;
                removed_labels.push(trimmed.to_string());
            }
        }

        let kept: Vec<&str> = segments
            .iter()
            .enumerate()
            .filter(|(i, _)| !removed_indices[*i])
            .map(|(_, s)| *s)
            .collect();

        // Rejoin with 、, then clean up leading/trailing commas and double commas
        let mut result = kept.join("、");
        // Remove leading comma
        while result.starts_with('、') {
            result = result[3..].to_string(); // 、 is 3 bytes in UTF-8
        }
        // Remove double commas left by removed segments
        while result.contains("、、") {
            result = result.replace("、、", "、");
        }

        Ok(FilterResult {
            text: result.trim().to_string(),
            removed: removed_labels,
        })
    }
}

impl JapaneseFillerFilter {
    /// Run the comma-segmented filler detector and return the
    /// codepoint span of every removed segment. Mirrors the Spanish
    /// filter's `detect_spans` so the L3 direct-F1 evaluator can
    /// compare emitted spans against the gold annotations in
    /// `tests/evaluation/annotations/ja_filler.jsonl`.
    pub fn detect_spans(&self, text: &str) -> Vec<Span> {
        // Tokenise on 、, tracking each segment's codepoint span in
        // the original text. The trimmed inner range is what the F1
        // evaluator scores against, so we record both the outer
        // segment boundaries and the inner trimmed boundaries.
        let chars: Vec<char> = text.chars().collect();
        let n_chars = chars.len();

        struct Seg {
            cp_inner_start: usize,
            cp_inner_end: usize,
            trimmed: String,
            ends_with_terminal: bool,
        }
        let mut segs: Vec<Seg> = Vec::new();

        let mut seg_start = 0usize;
        let mut j = 0;
        loop {
            let at_terminator = j == n_chars || chars[j] == '、';
            if at_terminator {
                let inner = &chars[seg_start..j];
                let mut tl = 0;
                while tl < inner.len() && inner[tl].is_whitespace() {
                    tl += 1;
                }
                let mut tr = inner.len();
                while tr > tl && inner[tr - 1].is_whitespace() {
                    tr -= 1;
                }
                let trimmed: String = inner[tl..tr].iter().collect();
                let ends_with_terminal = trimmed.ends_with('。');
                segs.push(Seg {
                    cp_inner_start: seg_start + tl,
                    cp_inner_end: seg_start + tr,
                    trimmed,
                    ends_with_terminal,
                });
                if j == n_chars {
                    break;
                }
                seg_start = j + 1; // skip past 、
            }
            j += 1;
        }

        let n = segs.len();
        let mut removed = vec![false; n];
        let mut detections: Vec<usize> = Vec::new();

        // Pass 1: pure fillers — unconditional.
        for i in 0..n {
            if segs[i].trimmed.is_empty() {
                continue;
            }
            if self.pure_fillers.iter().any(|f| segs[i].trimmed == *f) {
                removed[i] = true;
                detections.push(i);
            }
        }

        // Pass 2: contextual fillers at sentence-initial position or
        // immediately after 。.
        for i in 0..n {
            if removed[i] || segs[i].trimmed.is_empty() {
                continue;
            }
            let is_initial = (0..i).all(|k| removed[k] || segs[k].trimmed.is_empty());
            let after_period = i > 0 && !removed[i - 1] && segs[i - 1].ends_with_terminal;
            if self
                .contextual_fillers
                .iter()
                .any(|f| segs[i].trimmed == *f)
                && (is_initial || after_period)
            {
                removed[i] = true;
                detections.push(i);
            }
        }

        // Pass 3: standalone contextual fillers — segment IS the
        // filler word in its entirety.
        for i in 0..n {
            if removed[i] || segs[i].trimmed.is_empty() {
                continue;
            }
            if self
                .contextual_fillers
                .iter()
                .any(|f| segs[i].trimmed == *f)
            {
                removed[i] = true;
                detections.push(i);
            }
        }

        let mut spans: Vec<Span> = detections
            .into_iter()
            .map(|i| crate::types::Span {
                start: segs[i].cp_inner_start,
                end: segs[i].cp_inner_end,
            })
            .collect();
        spans.sort_by_key(|s| s.start);
        spans
    }
}

// ---------------------------------------------------------------------------
// ChineseFillerFilter — Mandarin Chinese rule-based filter
// ---------------------------------------------------------------------------

/// Rule-based filter for Mandarin Chinese filler tokens.
///
/// Closely mirrors `JapaneseFillerFilter`: splits the input by Chinese
/// commas (`，`), then applies a 3-pass rule set per segment:
///
/// 1. **Pure fillers** — `嗯`, `呃`, `哦`, etc. Always removed.
/// 2. **Contextual fillers** — `那个`, `这个`, `就是`, `然后`, `怎么说`.
///    These are also content words in their non-filler use (`那个`
///    "that one", `这个` "this one", `然后` "and then") so we only
///    remove them when they appear at sentence-initial position or
///    just after a `。` sentence boundary.
/// 3. **Standalone contextual fillers** — when a contextual filler
///    forms the entire `，`-delimited segment (e.g. "那个，我们需要…"
///    → segment 0 is the literal string "那个"), it's almost certainly
///    a filler, not a demonstrative.
///
/// The closed-set lexicon is informed by the Mandarin disfluency
/// literature surveyed in `docs/evaluation.md` §5.6.2 — well-attested
/// surface forms with high inter-annotator agreement (κ 0.80–0.95).
/// `啊` is intentionally **not** included as a pure filler because it
/// frequently appears as a sentence-final particle that's part of the
/// content; downgrading that judgement to false-positive territory
/// is worse than leaving it in.
#[derive(Debug, Clone)]
pub struct ChineseFillerFilter {
    /// Always removed regardless of position.
    pure_fillers: Vec<String>,
    /// Removed only at sentence-initial position or as a standalone segment.
    contextual_fillers: Vec<String>,
}

impl ChineseFillerFilter {
    pub fn new() -> Self {
        Self {
            pure_fillers: vec!["嗯", "呃", "哦", "唉", "呀"]
                .into_iter()
                .map(String::from)
                .collect(),
            contextual_fillers: vec!["那个", "这个", "就是", "然后", "怎么说"]
                .into_iter()
                .map(String::from)
                .collect(),
        }
    }
}

impl Default for ChineseFillerFilter {
    fn default() -> Self {
        Self::new()
    }
}

#[async_trait]
impl TextFilter for ChineseFillerFilter {
    async fn filter(&self, text: &str) -> Result<FilterResult, FilterError> {
        // Split by Chinese full-width comma. Half-width "," is rare in
        // Whisper's zh output but if it appears, we treat it the same.
        let normalized = text.replace(',', "，");
        let segments: Vec<&str> = normalized.split('，').collect();
        let n = segments.len();
        let mut removed_indices = vec![false; n];
        let mut removed_labels: Vec<String> = Vec::new();

        // Pass 1: pure fillers — remove unconditionally.
        for i in 0..n {
            let trimmed = segments[i].trim();
            if trimmed.is_empty() {
                continue;
            }
            if self.pure_fillers.iter().any(|f| trimmed == f) {
                removed_indices[i] = true;
                removed_labels.push(trimmed.to_string());
            }
        }

        // Pass 2: contextual fillers at sentence-initial position.
        for i in 0..n {
            if removed_indices[i] {
                continue;
            }
            let trimmed = segments[i].trim();
            if trimmed.is_empty() {
                continue;
            }
            let is_initial = (0..i).all(|j| removed_indices[j] || segments[j].trim().is_empty());
            let after_period =
                i > 0 && !removed_indices[i - 1] && segments[i - 1].trim().ends_with('。');

            if self.contextual_fillers.iter().any(|f| trimmed == f) && (is_initial || after_period)
            {
                removed_indices[i] = true;
                removed_labels.push(trimmed.to_string());
            }
        }

        // Pass 3: standalone contextual fillers (the whole segment IS
        // the filler word). 「那个、我们需要…」 vs. 「那个人」.
        for i in 0..n {
            if removed_indices[i] {
                continue;
            }
            let trimmed = segments[i].trim();
            if trimmed.is_empty() {
                continue;
            }
            if self.contextual_fillers.iter().any(|f| trimmed == f) {
                removed_indices[i] = true;
                removed_labels.push(trimmed.to_string());
            }
        }

        let kept: Vec<&str> = segments
            .iter()
            .enumerate()
            .filter(|(i, _)| !removed_indices[*i])
            .map(|(_, s)| *s)
            .collect();

        // Rejoin with 「，」, then clean up. Both `、` and `，` are
        // 3 bytes in UTF-8 so the byte-skip from JapaneseFillerFilter
        // applies directly.
        let mut result = kept.join("，");
        while result.starts_with('，') {
            result = result[3..].to_string();
        }
        while result.contains("，，") {
            result = result.replace("，，", "，");
        }

        Ok(FilterResult {
            text: result.trim().to_string(),
            removed: removed_labels,
        })
    }
}

impl ChineseFillerFilter {
    /// Run the comma-segmented filler detector and return the
    /// codepoint span of every removed segment. Mirrors the
    /// Japanese filter's `detect_spans` with `，` as the segment
    /// terminator instead of `、`. Used by the L3 direct-F1
    /// evaluator in `eval_l3 --task filler --lang zh`.
    pub fn detect_spans(&self, text: &str) -> Vec<Span> {
        let chars: Vec<char> = text.chars().collect();
        let n_chars = chars.len();

        struct Seg {
            cp_inner_start: usize,
            cp_inner_end: usize,
            trimmed: String,
            ends_with_terminal: bool,
        }
        let mut segs: Vec<Seg> = Vec::new();

        let mut seg_start = 0usize;
        let mut j = 0;
        loop {
            let at_terminator = j == n_chars || chars[j] == '，';
            if at_terminator {
                let inner = &chars[seg_start..j];
                let mut tl = 0;
                while tl < inner.len() && inner[tl].is_whitespace() {
                    tl += 1;
                }
                let mut tr = inner.len();
                while tr > tl && inner[tr - 1].is_whitespace() {
                    tr -= 1;
                }
                let trimmed: String = inner[tl..tr].iter().collect();
                let ends_with_terminal = trimmed.ends_with('。');
                segs.push(Seg {
                    cp_inner_start: seg_start + tl,
                    cp_inner_end: seg_start + tr,
                    trimmed,
                    ends_with_terminal,
                });
                if j == n_chars {
                    break;
                }
                seg_start = j + 1;
            }
            j += 1;
        }

        let n = segs.len();
        let mut removed = vec![false; n];
        let mut detections: Vec<usize> = Vec::new();

        // Pass 1: pure fillers — unconditional.
        for i in 0..n {
            if segs[i].trimmed.is_empty() {
                continue;
            }
            if self.pure_fillers.iter().any(|f| segs[i].trimmed == *f) {
                removed[i] = true;
                detections.push(i);
            }
        }

        // Pass 2: contextual fillers at sentence-initial position or
        // immediately after 。.
        for i in 0..n {
            if removed[i] || segs[i].trimmed.is_empty() {
                continue;
            }
            let is_initial = (0..i).all(|k| removed[k] || segs[k].trimmed.is_empty());
            let after_period = i > 0 && !removed[i - 1] && segs[i - 1].ends_with_terminal;
            if self
                .contextual_fillers
                .iter()
                .any(|f| segs[i].trimmed == *f)
                && (is_initial || after_period)
            {
                removed[i] = true;
                detections.push(i);
            }
        }

        // Pass 3: standalone contextual fillers — segment IS the
        // filler in its entirety.
        for i in 0..n {
            if removed[i] || segs[i].trimmed.is_empty() {
                continue;
            }
            if self
                .contextual_fillers
                .iter()
                .any(|f| segs[i].trimmed == *f)
            {
                removed[i] = true;
                detections.push(i);
            }
        }

        let mut spans: Vec<Span> = detections
            .into_iter()
            .map(|i| crate::types::Span {
                start: segs[i].cp_inner_start,
                end: segs[i].cp_inner_end,
            })
            .collect();
        spans.sort_by_key(|s| s.start);
        spans
    }
}

// ---------------------------------------------------------------------------
// SpanishFillerFilter — token-level rule-based filter for Spanish
// ---------------------------------------------------------------------------

/// Rule-based filler removal for Spanish text from ASR output.
///
/// Spanish uses whitespace tokenization (unlike `JapaneseFillerFilter` /
/// `ChineseFillerFilter` which segment on `、` / `，`) so the token-pass
/// structure mirrors `SimpleFillerFilter`. Three Spanish-specific
/// patterns drive the design beyond the en `pure / contextual / multi`
/// schema:
///
/// 1. **Token repetitions** — `del del`, `que que`. Removed by flagging
///    the earlier occurrence so the canonical surface form survives.
/// 2. **2-token bigram repetitions** — `a la a la`. Take precedence over
///    the 1-token pass so `a la` doesn't degenerate into per-token
///    flags that miss the pattern.
/// 3. **Partial / abandoned words** — `sie siempre`, `est este`: a
///    truncated start-of-word that the speaker abandons and re-starts.
///    Detected by strict-prefix relation between adjacent tokens with a
///    minimum-length guard. Common Spanish function words (`de`, `la`,
///    `el`, …) are stoplisted to keep precision high without POS tags.
///
/// Closed-class lexicons mirror `scripts/build_es_filler_annotations.py`
/// so that the filter's deletions align span-for-span with the F1
/// gold standard built from CIEMPIESS Test transcripts.
#[derive(Debug, Clone)]
pub struct SpanishFillerFilter {
    /// Always removed regardless of position.
    pure_fillers: Vec<String>,
    /// Multi-word fillers (lowercased token sequences).
    multi_fillers: Vec<Vec<String>>,
    /// Tokens excluded from the partial-word pass — common short
    /// Spanish words that legitimately precede longer words.
    partial_stoplist: Vec<String>,
}

impl SpanishFillerFilter {
    pub fn new() -> Self {
        Self {
            pure_fillers: vec![
                // vowel-only hesitations (lone "e" is the dominant
                // CIEMPIESS hesitation form)
                "e", "eh", "ehh", "ehhh", "eee", "eeh", "eeeh", "ehm",
                // nasal hesitations
                "mm", "mmm", "hmm", // "ah" / "oh" family
                "ah", "aah", "ahh", "oh",
            ]
            .into_iter()
            .map(String::from)
            .collect(),
            multi_fillers: vec![vec!["o".into(), "sea".into()]],
            partial_stoplist: vec![
                "y", "a", "o", "u", "e", "de", "en", "el", "la", "lo", "los", "las", "le", "les",
                "un", "una", "uno", "unos", "unas", "se", "te", "me", "nos", "os", "no", "ni",
                "es", "ya", "si", "sí", "por", "con", "para", "del", "al", "que", "qué", "su",
                "sus", "mi", "mis", "tu", "tus",
            ]
            .into_iter()
            .map(String::from)
            .collect(),
        }
    }
}

impl Default for SpanishFillerFilter {
    fn default() -> Self {
        Self::new()
    }
}

/// Internal token shape carrying both lowercase form and char-offset
/// span. We track codepoint offsets explicitly so the F1 evaluator
/// can compare against the codepoint-indexed annotation schema in
/// `tests/evaluation/annotations/`.
struct EsToken {
    surface: String,
    lower: String,
    char_count: usize,
    char_start: usize,
    char_end: usize,
}

fn tokenize_es(text: &str) -> Vec<EsToken> {
    let chars: Vec<char> = text.chars().collect();
    let n = chars.len();
    let mut out = Vec::new();
    let mut i = 0;
    while i < n {
        while i < n && chars[i].is_whitespace() {
            i += 1;
        }
        if i >= n {
            break;
        }
        let start = i;
        while i < n && !chars[i].is_whitespace() {
            i += 1;
        }
        let surface: String = chars[start..i].iter().collect();
        // Match on the word alone. Spanish tokenises on whitespace, so
        // without this `"eh,"` never equals the `"eh"` lexicon entry and a
        // punctuated filler survives untouched. `surface` and the char
        // offsets keep the punctuation, so a removed token takes its
        // trailing comma with it — the same result the English filter
        // produces via `take_while(is_alphanumeric)`.
        let lower = surface
            .to_lowercase()
            .trim_matches(|c: char| !c.is_alphanumeric())
            .to_string();
        let char_count = lower.chars().count();
        out.push(EsToken {
            surface,
            lower,
            char_count,
            char_start: start,
            char_end: i,
        });
    }
    out
}

/// One filler detection: an inclusive token range `[start_token,
/// end_token]`. Multi-token detections (multi-word filler "o sea",
/// 2-token rep "a la") have `end_token > start_token`; single-token
/// detections have `start_token == end_token`.
struct EsDetection {
    start_token: usize,
    end_token: usize,
}

impl SpanishFillerFilter {
    /// Run the 5-pass detector. Returns `(removed_flags, detections)`.
    /// Detections are emitted at most once per pattern so `detect_spans`
    /// and `filter().removed` can both be derived without re-merging.
    fn run_passes(&self, tokens: &[EsToken]) -> (Vec<bool>, Vec<EsDetection>) {
        let n = tokens.len();
        let mut removed = vec![false; n];
        let mut detections: Vec<EsDetection> = Vec::new();

        // Pass 1: multi-word fillers (e.g. "o sea").
        for mf in &self.multi_fillers {
            let plen = mf.len();
            if plen == 0 || plen > n {
                continue;
            }
            let mut i = 0;
            while i + plen <= n {
                let matches = (0..plen).all(|k| !removed[i + k] && tokens[i + k].lower == mf[k]);
                if matches {
                    detections.push(EsDetection {
                        start_token: i,
                        end_token: i + plen - 1,
                    });
                    for k in 0..plen {
                        removed[i + k] = true;
                    }
                    i += plen;
                } else {
                    i += 1;
                }
            }
        }

        // Pass 2: 2-token bigram repetitions ("a la a la"). Take this
        // before the 1-token pass so a bigram pattern is recognised
        // as a single unit and the surviving copy stays intact.
        if n >= 4 {
            let mut i = 0;
            while i + 4 <= n {
                let any_covered = (0..4).any(|k| removed[i + k]);
                if !any_covered
                    && tokens[i].lower == tokens[i + 2].lower
                    && tokens[i + 1].lower == tokens[i + 3].lower
                    && tokens[i].char_count + tokens[i + 1].char_count >= 3
                {
                    detections.push(EsDetection {
                        start_token: i,
                        end_token: i + 1,
                    });
                    removed[i] = true;
                    removed[i + 1] = true;
                    // skip the kept pair; further matches start after it
                    i += 2;
                } else {
                    i += 1;
                }
            }
        }

        // Pass 3: 1-token immediate repetitions. Flag the EARLIER
        // occurrence so the canonical surface form survives.
        for j in 1..n {
            if removed[j] || removed[j - 1] {
                continue;
            }
            if tokens[j].lower == tokens[j - 1].lower && tokens[j].char_count >= 2 {
                removed[j - 1] = true;
                detections.push(EsDetection {
                    start_token: j - 1,
                    end_token: j - 1,
                });
            }
        }

        // Pass 4: partial / abandoned words. Flag a token when the
        // following token has it as a strict prefix with a length gap
        // of ≥ 2 chars (so legitimate plural / inflected pairs like
        // `mes / meses` aren't caught), and the prefix isn't on the
        // function-word stoplist.
        if n >= 2 {
            for j in 0..n - 1 {
                if removed[j] || removed[j + 1] {
                    continue;
                }
                let a = &tokens[j].lower;
                let b = &tokens[j + 1].lower;
                if tokens[j].char_count >= 2
                    && tokens[j + 1].char_count >= tokens[j].char_count + 2
                    && b.starts_with(a.as_str())
                    && !self.partial_stoplist.contains(a)
                {
                    removed[j] = true;
                    detections.push(EsDetection {
                        start_token: j,
                        end_token: j,
                    });
                }
            }
        }

        // Pass 5: pure fillers (closed lex).
        for j in 0..n {
            if removed[j] {
                continue;
            }
            if self.pure_fillers.contains(&tokens[j].lower) {
                removed[j] = true;
                detections.push(EsDetection {
                    start_token: j,
                    end_token: j,
                });
            }
        }

        (removed, detections)
    }

    /// Detect filler spans without rewriting the text. Returns
    /// **codepoint-offset** half-open `[start, end)` ranges, sorted by
    /// `start`, suitable for direct comparison against
    /// `tests/evaluation/annotations/*.jsonl` filler entries.
    pub fn detect_spans(&self, text: &str) -> Vec<Span> {
        let tokens = tokenize_es(text);
        let (_removed, detections) = self.run_passes(&tokens);
        let mut spans: Vec<Span> = detections
            .iter()
            .map(|d| crate::types::Span {
                start: tokens[d.start_token].char_start,
                end: tokens[d.end_token].char_end,
            })
            .collect();
        spans.sort_by_key(|s| s.start);
        spans
    }

    fn detection_label(&self, tokens: &[EsToken], d: &EsDetection) -> String {
        if d.start_token == d.end_token {
            tokens[d.start_token].surface.clone()
        } else {
            (d.start_token..=d.end_token)
                .map(|i| tokens[i].surface.as_str())
                .collect::<Vec<_>>()
                .join(" ")
        }
    }
}

#[async_trait]
impl TextFilter for SpanishFillerFilter {
    async fn filter(&self, text: &str) -> Result<FilterResult, FilterError> {
        let tokens = tokenize_es(text);
        let (removed, mut detections) = self.run_passes(&tokens);
        // emit labels in text order so callers see them left-to-right
        detections.sort_by_key(|d| d.start_token);
        let removed_labels: Vec<String> = detections
            .iter()
            .map(|d| self.detection_label(&tokens, d))
            .collect();

        let kept: Vec<&str> = tokens
            .iter()
            .enumerate()
            .filter(|(i, _)| !removed[*i])
            .map(|(_, t)| t.surface.as_str())
            .collect();

        Ok(FilterResult {
            text: kept.join(" "),
            removed: removed_labels,
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn simple_filler_removes_or_something_bigram() {
        // Ported from the retired Python `filler_filter.py` lexicon,
        // which carried two multi-word entries the Rust filter lacked.
        let filter = SimpleFillerFilter::english();
        let result = filter.filter("we should deploy or something").await.unwrap();
        assert_eq!(result.text, "we should deploy");
        assert!(result.removed.iter().any(|r| r.to_lowercase().contains("or")));
    }

    #[tokio::test]
    async fn simple_filler_removes_or_whatever_bigram() {
        let filter = SimpleFillerFilter::english();
        let result = filter.filter("restart the service or whatever").await.unwrap();
        assert_eq!(result.text, "restart the service");
    }

    #[tokio::test]
    async fn simple_filler_keeps_or_when_not_part_of_a_filler_bigram() {
        // Guard against the substring matching the Python path used,
        // which would fire on word-internal matches.
        let filter = SimpleFillerFilter::english();
        let result = filter.filter("deploy or roll back").await.unwrap();
        assert_eq!(result.text, "deploy or roll back");
        assert!(result.removed.is_empty());
    }

    #[tokio::test]
    async fn simple_filler_filter_removes_pure_fillers() {
        let filter = SimpleFillerFilter::english();
        let result = filter
            .filter("um I think uh we should deploy")
            .await
            .unwrap();
        assert_eq!(result.text, "I think we should deploy");
        assert!(result.removed.contains(&"um".to_string()));
        assert!(result.removed.contains(&"uh".to_string()));
    }

    #[tokio::test]
    async fn simple_filler_filter_preserves_clean_text() {
        let filter = SimpleFillerFilter::english();
        let result = filter.filter("deploy the server now").await.unwrap();
        assert_eq!(result.text, "deploy the server now");
        assert!(result.removed.is_empty());
    }

    #[tokio::test]
    async fn contextual_filler_removed_at_sentence_start() {
        let filter = SimpleFillerFilter::english();
        let result = filter.filter("So I think we should deploy").await.unwrap();
        assert_eq!(result.text, "I think we should deploy");
        assert!(result.removed.contains(&"So".to_string()));
    }

    #[tokio::test]
    async fn contextual_word_preserved_mid_sentence() {
        let filter = SimpleFillerFilter::english();
        // "so" as conjunction mid-sentence should be kept
        let result = filter.filter("I think so we should deploy").await.unwrap();
        assert!(result.text.contains("so"));
    }

    #[tokio::test]
    async fn well_preserved_as_content_word() {
        let filter = SimpleFillerFilter::english();
        // "well" as adverb mid-sentence
        let result = filter.filter("it went well today").await.unwrap();
        assert_eq!(result.text, "it went well today");
    }

    #[tokio::test]
    async fn well_removed_at_sentence_start() {
        let filter = SimpleFillerFilter::english();
        let result = filter
            .filter("Well I think we should deploy")
            .await
            .unwrap();
        assert_eq!(result.text, "I think we should deploy");
    }

    #[tokio::test]
    async fn multi_word_filler_removed() {
        let filter = SimpleFillerFilter::english();
        let result = filter
            .filter("I think you know we should deploy")
            .await
            .unwrap();
        assert_eq!(result.text, "I think we should deploy");
        assert!(result.removed.contains(&"you know".to_string()));
    }

    /// Regression: a multi-word filler whose match leaves `i` exactly at
    /// the end of the word list used to fall through to the next mf in
    /// the inner `for` loop and index `removed_indices[i + 1]`
    /// out-of-bounds.
    #[tokio::test]
    async fn multi_word_filler_at_end_of_input() {
        let filter = SimpleFillerFilter::english();
        let result = filter.filter("deploy now you know").await.unwrap();
        assert!(!result.text.contains("you know"), "{}", result.text);
        assert!(result.removed.contains(&"you know".to_string()));
    }

    #[tokio::test]
    async fn combined_pure_contextual_and_multi() {
        let filter = SimpleFillerFilter::english();
        let result = filter
            .filter("Well um I think you know we should uh deploy")
            .await
            .unwrap();
        assert_eq!(result.text, "I think we should deploy");
    }

    #[tokio::test]
    async fn jfk_speech_contextual_so_preserved() {
        let filter = SimpleFillerFilter::english();
        // "And so" — "so" follows "And" which is a kept word mid-sentence,
        // so it should be preserved
        let result = filter
            .filter("And so my fellow Americans ask not what your country can do for you")
            .await
            .unwrap();
        assert!(
            result.text.contains("so"),
            "mid-sentence 'so' should be kept: {}",
            result.text
        );
    }

    // --- Japanese filler filter tests ---

    #[tokio::test]
    async fn japanese_pure_filler_removed() {
        let filter = JapaneseFillerFilter::new();
        let result = filter.filter("えーと、拙者親方と申すは").await.unwrap();
        assert_eq!(result.text, "拙者親方と申すは");
        assert!(result.removed.contains(&"えーと".to_string()));
    }

    #[tokio::test]
    async fn japanese_contextual_filler_at_start() {
        let filter = JapaneseFillerFilter::new();
        let result = filter.filter("あの、お立ち合いの中に").await.unwrap();
        assert_eq!(result.text, "お立ち合いの中に");
        assert!(result.removed.contains(&"あの".to_string()));
    }

    #[tokio::test]
    async fn japanese_contextual_filler_standalone_removed() {
        let filter = JapaneseFillerFilter::new();
        // "あの" as standalone segment between commas → filler, removed
        let result = filter
            .filter("拙者親方と申すは、あの、お立ち合いの中に")
            .await
            .unwrap();
        assert!(
            !result.text.contains("あの"),
            "standalone あの should be removed: {}",
            result.text
        );
        assert!(result.removed.contains(&"あの".to_string()));
    }

    #[tokio::test]
    async fn japanese_contextual_word_in_phrase_preserved() {
        let filter = JapaneseFillerFilter::new();
        // "あの" as part of a longer phrase → demonstrative, preserved
        let result = filter
            .filter("拙者親方と申すは、あの人が来た")
            .await
            .unwrap();
        assert!(
            result.text.contains("あの人が来た"),
            "あの in phrase should be kept: {}",
            result.text
        );
    }

    #[tokio::test]
    async fn japanese_multiple_fillers() {
        let filter = JapaneseFillerFilter::new();
        let result = filter
            .filter("えーと、あの、お江戸を発って、えー、二十里上方")
            .await
            .unwrap();
        assert!(result.removed.contains(&"えーと".to_string()));
        assert!(result.removed.contains(&"えー".to_string()));
        assert!(result.text.contains("お江戸を発って"));
        assert!(result.text.contains("二十里上方"));
    }

    #[tokio::test]
    async fn japanese_asr_artifact_removed() {
        let filter = JapaneseFillerFilter::new();
        let result = filter.filter("映像、拙者親方と申すは").await.unwrap();
        assert_eq!(result.text, "拙者親方と申すは");
        assert!(result.removed.contains(&"映像".to_string()));
    }

    #[tokio::test]
    async fn japanese_clean_text_preserved() {
        let filter = JapaneseFillerFilter::new();
        let result = filter
            .filter("拙者親方と申すは、お立ち合いの中に、御存じのお方もございましょうが")
            .await
            .unwrap();
        assert_eq!(
            result.text,
            "拙者親方と申すは、お立ち合いの中に、御存じのお方もございましょうが"
        );
        assert!(result.removed.is_empty());
    }

    #[tokio::test]
    async fn japanese_uiro_with_fillers_full() {
        let filter = JapaneseFillerFilter::new();
        let result = filter
            .filter("映像、せっしゃおやかたとモースは、あの、おたち合いの中に、まあ、ご存知のおかたもございましょうが、その、お江戸をたってにじゅうりかみがた")
            .await
            .unwrap();
        assert!(
            !result.text.contains("映像"),
            "ASR artifact should be removed"
        );
        assert!(result.text.contains("せっしゃおやかたとモースは"));
        assert!(result.text.contains("ご存知のおかたもございましょうが"));
    }

    // -----------------------------------------------------------------
    // ChineseFillerFilter
    // -----------------------------------------------------------------

    #[tokio::test]
    async fn chinese_pure_filler_removed() {
        let filter = ChineseFillerFilter::new();
        let result = filter.filter("嗯，我们需要更新数据库").await.unwrap();
        assert_eq!(result.text, "我们需要更新数据库");
        assert!(result.removed.contains(&"嗯".to_string()));
    }

    #[tokio::test]
    async fn chinese_two_pure_fillers() {
        let filter = ChineseFillerFilter::new();
        let result = filter.filter("呃，明天发布新版本，哦").await.unwrap();
        assert!(!result.text.contains("呃"), "{}", result.text);
        assert!(!result.text.contains("哦"), "{}", result.text);
        assert!(result.text.contains("明天发布新版本"));
    }

    #[tokio::test]
    async fn chinese_contextual_filler_at_sentence_start() {
        let filter = ChineseFillerFilter::new();
        let result = filter.filter("那个，测试全部通过了").await.unwrap();
        assert_eq!(result.text, "测试全部通过了");
        assert!(result.removed.contains(&"那个".to_string()));
    }

    #[tokio::test]
    async fn chinese_contextual_word_in_phrase_preserved() {
        // "那个" inside a noun phrase ("那个人" — "that person") must
        // NOT be removed; only the standalone-segment form is a filler.
        let filter = ChineseFillerFilter::new();
        let result = filter.filter("那个人是谁").await.unwrap();
        assert_eq!(result.text, "那个人是谁");
        assert!(result.removed.is_empty());
    }

    #[tokio::test]
    async fn chinese_standalone_contextual_filler_mid_sentence() {
        // 「那个」 as its own 「，」-delimited segment — even mid-sentence
        // — is the filler usage (Pass 3).
        let filter = ChineseFillerFilter::new();
        let result = filter
            .filter("我觉得，那个，这个事情应该这样处理")
            .await
            .unwrap();
        assert!(!result.text.contains("，那个，"), "{}", result.text);
        assert!(result.text.contains("我觉得"));
        assert!(result.text.contains("这个事情应该这样处理"));
    }

    #[tokio::test]
    async fn chinese_clean_text_preserved() {
        let filter = ChineseFillerFilter::new();
        let result = filter
            .filter("配置文件已经修改，缓存命中率不错")
            .await
            .unwrap();
        assert_eq!(result.text, "配置文件已经修改，缓存命中率不错");
        assert!(result.removed.is_empty());
    }

    #[tokio::test]
    async fn chinese_halfwidth_comma_normalised() {
        // Whisper for zh almost always emits 「，」 but if a half-width
        // comma slips through we should still segment correctly.
        let filter = ChineseFillerFilter::new();
        let result = filter.filter("嗯,我们需要更新数据库").await.unwrap();
        assert!(!result.text.contains("嗯"));
        assert!(result.text.contains("我们需要更新数据库"));
    }

    #[tokio::test]
    async fn chinese_contextual_after_sentence_end() {
        // After a 。 sentence break the contextual filler appears as
        // its own 「，」-delimited segment — Pass 2's after_period
        // branch picks it up.
        let filter = ChineseFillerFilter::new();
        let result = filter
            .filter("第一个任务完成。，然后，开始第二个")
            .await
            .unwrap();
        assert!(!result.text.contains("然后"), "{}", result.text);
        assert!(result.text.contains("开始第二个"));
    }

    // -----------------------------------------------------------------
    // SpanishFillerFilter
    // -----------------------------------------------------------------

    #[tokio::test]
    async fn spanish_pure_filler_removed_at_start() {
        // Lone "e" is the dominant CIEMPIESS hesitation form.
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("e fuera del aire").await.unwrap();
        assert_eq!(result.text, "fuera del aire");
        assert!(result.removed.contains(&"e".to_string()));
    }

    #[tokio::test]
    async fn spanish_pure_filler_mid_utterance() {
        let filter = SpanishFillerFilter::new();
        let result = filter
            .filter("hace rato comentaba e fuera del aire")
            .await
            .unwrap();
        assert!(
            !result.text.split_whitespace().any(|t| t == "e"),
            "{}",
            result.text
        );
        assert!(result.text.contains("comentaba"));
        assert!(result.text.contains("fuera del aire"));
    }

    #[tokio::test]
    async fn spanish_multi_pure_fillers() {
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("eh hola eee mañana").await.unwrap();
        assert_eq!(result.text, "hola mañana");
    }

    #[tokio::test]
    async fn spanish_pure_filler_case_insensitive() {
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("Eh hola").await.unwrap();
        assert_eq!(result.text, "hola");
        assert!(result.removed.contains(&"Eh".to_string()));
    }

    #[tokio::test]
    async fn spanish_multi_word_filler_o_sea() {
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("o sea ahora las mujeres").await.unwrap();
        assert_eq!(result.text, "ahora las mujeres");
        assert!(result.removed.contains(&"o sea".to_string()));
    }

    #[tokio::test]
    async fn spanish_token_repetition_drops_first() {
        // "del del" — flag the EARLIER occurrence so the canonical
        // surface form survives.
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("e fuera del del aire").await.unwrap();
        assert_eq!(result.text, "fuera del aire");
        assert!(result.removed.contains(&"e".to_string()));
        assert!(result.removed.contains(&"del".to_string()));
    }

    #[tokio::test]
    async fn spanish_three_fold_repetition_keeps_last() {
        // "muy muy muy" — first two are disfluency, last one is the
        // canonical form.
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("muy muy muy ligada con").await.unwrap();
        assert_eq!(result.text, "muy ligada con");
    }

    #[tokio::test]
    async fn spanish_two_token_repetition_drops_first_pair() {
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("a la a la casa de mi abuela").await.unwrap();
        assert_eq!(result.text, "a la casa de mi abuela");
        assert!(result.removed.contains(&"a la".to_string()));
    }

    #[tokio::test]
    async fn spanish_partial_word_dropped() {
        // "sie" is a strict prefix of "siempre" with len ≥ 2 and a
        // gap of ≥ 2 chars → flagged as an abandoned-word disfluency.
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("sie siempre estuvo leyendo").await.unwrap();
        assert_eq!(result.text, "siempre estuvo leyendo");
        assert!(result.removed.contains(&"sie".to_string()));
    }

    #[tokio::test]
    async fn spanish_partial_stoplist_protects_function_words() {
        // "se" is a stoplisted function word — even though "se" is a
        // strict prefix of "sevir" we must not flag it.
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("se sevir mañana").await.unwrap();
        assert_eq!(result.text, "se sevir mañana");
        assert!(result.removed.is_empty());
    }

    #[tokio::test]
    async fn spanish_repetition_of_contextual_word_dropped() {
        // "este este libro" — repetition pass fires; the bare phrase
        // "este libro" stays clean (covered by the next test).
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("este este libro").await.unwrap();
        assert_eq!(result.text, "este libro");
        assert!(result.removed.contains(&"este".to_string()));
    }

    #[tokio::test]
    async fn spanish_clean_demonstrative_preserved() {
        // Bare "este libro" must not be flagged — this is the false-
        // positive guard that motivates leaving "este" out of the
        // pure-filler set in v1.
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("este libro es interesante").await.unwrap();
        assert_eq!(result.text, "este libro es interesante");
        assert!(result.removed.is_empty());
    }

    #[tokio::test]
    async fn spanish_clean_text_preserved() {
        let filter = SpanishFillerFilter::new();
        let result = filter
            .filter("como una de las partes importantes es el capitulado")
            .await
            .unwrap();
        assert_eq!(
            result.text,
            "como una de las partes importantes es el capitulado"
        );
        assert!(result.removed.is_empty());
    }

    #[tokio::test]
    async fn spanish_accented_characters_preserved() {
        // Spans / tokens must align correctly when the sentence
        // contains multi-byte UTF-8 codepoints.
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("e canción mañana").await.unwrap();
        assert_eq!(result.text, "canción mañana");
        assert!(result.removed.contains(&"e".to_string()));
    }

    #[tokio::test]
    async fn spanish_combined_pure_repetition_partial() {
        // CIEMPIESS-style multi-pattern utterance: pure filler at
        // start, repetition, partial word.
        let filter = SpanishFillerFilter::new();
        let result = filter
            .filter("e a la a la sie siempre estuvo")
            .await
            .unwrap();
        assert_eq!(result.text, "a la siempre estuvo");
    }

    #[tokio::test]
    async fn spanish_empty_text() {
        let filter = SpanishFillerFilter::new();
        let result = filter.filter("").await.unwrap();
        assert_eq!(result.text, "");
        assert!(result.removed.is_empty());
    }

    // --- detect_spans (char-offset span emission for Tier 1 F1) ---

    fn span(start: usize, end: usize) -> crate::types::Span {
        crate::types::Span { start, end }
    }

    #[test]
    fn spanish_detect_spans_pure_filler_at_start() {
        let filter = SpanishFillerFilter::new();
        // "e fuera" → span [0, 1) for "e"
        assert_eq!(filter.detect_spans("e fuera del aire"), vec![span(0, 1)]);
    }

    #[test]
    fn spanish_detect_spans_token_repetition() {
        let filter = SpanishFillerFilter::new();
        // "e fuera del del aire" → spans for "e" (0-1) and the first "del" (8-11)
        assert_eq!(
            filter.detect_spans("e fuera del del aire"),
            vec![span(0, 1), span(8, 11)]
        );
    }

    #[test]
    fn spanish_detect_spans_two_token_repetition() {
        let filter = SpanishFillerFilter::new();
        // "a la a la casa" → span [0, 4) covering the first "a la"
        assert_eq!(filter.detect_spans("a la a la casa"), vec![span(0, 4)]);
    }

    #[test]
    fn spanish_detect_spans_partial_word() {
        let filter = SpanishFillerFilter::new();
        // "sie siempre estuvo" → span [0, 3) for "sie"
        assert_eq!(filter.detect_spans("sie siempre estuvo"), vec![span(0, 3)]);
    }

    #[test]
    fn spanish_detect_spans_multi_word_filler() {
        let filter = SpanishFillerFilter::new();
        // "o sea ahora" → span [0, 5) covering "o sea"
        assert_eq!(
            filter.detect_spans("o sea ahora las mujeres"),
            vec![span(0, 5)]
        );
    }

    #[test]
    fn spanish_detect_spans_uses_codepoint_offsets() {
        // Accented codepoints would shift byte offsets but NOT char
        // offsets. Spans must be expressed in chars to match the
        // annotation schema in `tests/evaluation/annotations/`.
        let filter = SpanishFillerFilter::new();
        // "e canción mañana" — "e" at codepoint [0,1), then space at 1,
        // "canción" at [2, 9). Two ñ/ó codepoints encoded as 2 bytes
        // each in UTF-8 — verify we still get char offsets.
        assert_eq!(filter.detect_spans("e canción mañana"), vec![span(0, 1)]);
    }

    #[test]
    fn spanish_detect_spans_combined() {
        let filter = SpanishFillerFilter::new();
        // "e a la a la sie siempre estuvo"
        // codepoints:
        //   "e"        [0, 1)
        //   "a la a la" → first pair = "a la" [2, 6)
        //   "sie"      [12, 15)
        let spans = filter.detect_spans("e a la a la sie siempre estuvo");
        assert_eq!(spans, vec![span(0, 1), span(2, 6), span(12, 15)]);
    }

    #[test]
    fn spanish_detect_spans_clean_text_yields_empty() {
        let filter = SpanishFillerFilter::new();
        assert!(filter.detect_spans("este libro es interesante").is_empty());
    }

    #[test]
    fn spanish_detect_spans_empty_input() {
        let filter = SpanishFillerFilter::new();
        assert!(filter.detect_spans("").is_empty());
        assert!(filter.detect_spans("   ").is_empty());
    }

    #[test]
    fn spanish_detect_spans_match_filter_results() {
        // Invariant: spans returned by detect_spans should correspond
        // exactly to the surface tokens that filter() removes. Drives
        // alignment between filter and Tier 1 F1 evaluator.
        let filter = SpanishFillerFilter::new();
        let text = "e fuera del del a la a la casa";
        let spans = filter.detect_spans(text);

        let chars: Vec<char> = text.chars().collect();
        let removed_surfaces: Vec<String> = spans
            .iter()
            .map(|s| chars[s.start..s.end].iter().collect())
            .collect();

        let rt = tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()
            .unwrap();
        let result = rt.block_on(filter.filter(text)).unwrap();

        // detect_spans returns surface ranges; filter().removed lists
        // surface tokens. Same set, possibly different order.
        let mut a = removed_surfaces.clone();
        let mut b = result.removed.clone();
        a.sort();
        b.sort();
        assert_eq!(a, b);
    }

    // -----------------------------------------------------------------
    // Korean filler — full layer implementation parity with en / ja
    // / zh / es. Issue #47 follow-up.
    // -----------------------------------------------------------------

    #[tokio::test]
    async fn korean_pure_filler_removed_at_start() {
        let filter = SimpleFillerFilter::korean();
        let result = filter.filter("음 회의 시간을 변경합시다").await.unwrap();
        assert_eq!(result.text, "회의 시간을 변경합시다");
        assert!(result.removed.contains(&"음".to_string()));
    }

    #[tokio::test]
    async fn korean_pure_filler_removed_mid_utterance() {
        let filter = SimpleFillerFilter::korean();
        let result = filter.filter("회의 시간을 어 변경합시다").await.unwrap();
        // `어` is a pure filler — removed regardless of position.
        assert!(
            !result.text.split_whitespace().any(|t| t == "어"),
            "{}",
            result.text
        );
        assert!(result.text.contains("회의"));
        assert!(result.text.contains("변경합시다"));
    }

    #[tokio::test]
    async fn korean_contextual_filler_removed_at_sentence_start() {
        let filter = SimpleFillerFilter::korean();
        let result = filter.filter("그러니까 다음 주에 만나자").await.unwrap();
        assert_eq!(result.text, "다음 주에 만나자");
    }

    #[tokio::test]
    async fn korean_contextual_filler_kept_mid_utterance() {
        let filter = SimpleFillerFilter::korean();
        // `그` mid-sentence is a demonstrative ("that"), not a filler.
        let result = filter.filter("나는 그 사람을 만났다").await.unwrap();
        assert!(result.text.contains("그 사람"), "{}", result.text);
    }

    #[tokio::test]
    async fn korean_multi_word_filler_removed() {
        let filter = SimpleFillerFilter::korean();
        let result = filter.filter("음 그러니까 회의를 시작하자").await.unwrap();
        assert_eq!(result.text, "회의를 시작하자");
    }

    #[tokio::test]
    async fn korean_clean_text_unchanged() {
        let filter = SimpleFillerFilter::korean();
        let result = filter
            .filter("내일 오후 세 시에 만나기로 했습니다")
            .await
            .unwrap();
        assert_eq!(result.text, "내일 오후 세 시에 만나기로 했습니다");
    }

    // -----------------------------------------------------------------
    // FillerFilter::for_language
    // -----------------------------------------------------------------

    /// The bug this type exists to prevent. `SimpleFillerFilter` splits on
    /// whitespace, so Japanese text arrives as one token; when that token
    /// begins with a filler the whole utterance was deleted. Routing by
    /// `Language` must keep the content.
    #[tokio::test]
    async fn japanese_filler_removal_keeps_the_utterance() {
        let filter = FillerFilter::for_language(Language::Japanese);
        let result = filter
            .filter("えーと、今日は会議が三時からあります。")
            .await
            .unwrap();
        assert_eq!(result.text, "今日は会議が三時からあります。");
    }

    /// Same shape in Chinese, which is comma-segmented on ，rather than 、.
    #[tokio::test]
    async fn chinese_filler_removal_keeps_the_utterance() {
        let filter = FillerFilter::for_language(Language::Chinese);
        let result = filter.filter("嗯，我们明天开会。").await.unwrap();
        assert_eq!(result.text, "我们明天开会。");
    }

    /// Every language must actually remove its leading filler *and* keep
    /// the content. Asserting only the second half is what let a Spanish
    /// no-op hide: an utterance nothing is removed from is trivially
    /// non-empty. Both halves, across every language, in one place.
    #[tokio::test]
    async fn every_language_removes_the_filler_and_keeps_the_content() {
        let cases = [
            (
                Language::English,
                "um, the meeting is at three.",
                "the meeting is at three.",
            ),
            (
                Language::Japanese,
                "えーと、会議は三時からです。",
                "会議は三時からです。",
            ),
            (Language::Chinese, "嗯，会议在三点开始。", "会议在三点开始。"),
            (
                Language::Korean,
                "음 회의는 세 시에 시작합니다",
                "회의는 세 시에 시작합니다",
            ),
            (
                Language::Spanish,
                "eh, la reunión es a las tres.",
                "la reunión es a las tres.",
            ),
        ];
        for (language, input, expected) in cases {
            let result = FillerFilter::for_language(language)
                .filter(input)
                .await
                .unwrap();
            assert_eq!(result.text, expected, "{language:?} on {input:?}");
            assert!(
                !result.removed.is_empty(),
                "{language:?} removed nothing from {input:?}"
            );
        }
    }

    /// A filler followed by punctuation is still a filler. Spanish
    /// tokenises on whitespace alone, so `"eh,"` did not match the `"eh"`
    /// lexicon entry and the filter silently did nothing — the sibling
    /// filters all strip punctuation before matching.
    #[tokio::test]
    async fn spanish_filler_followed_by_punctuation_is_removed() {
        let filter = SpanishFillerFilter::new();

        let result = filter.filter("eh, la reunión es a las tres").await.unwrap();
        assert_eq!(result.text, "la reunión es a las tres");

        // Multi-word fillers take punctuation on the final token.
        let result = filter.filter("o sea, no lo sé").await.unwrap();
        assert_eq!(result.text, "no lo sé");

        // The dominant CIEMPIESS form, punctuated.
        let result = filter.filter("e, fuera del aire").await.unwrap();
        assert_eq!(result.text, "fuera del aire");
    }

    /// Text with no filler must survive byte-for-byte in every language —
    /// the filter is only allowed to remove what it identifies.
    #[tokio::test]
    async fn filler_free_text_is_untouched() {
        let cases = [
            (Language::English, "the meeting is at three."),
            (Language::Japanese, "今日は良い天気ですね。"),
            (Language::Chinese, "今天天气很好。"),
            (Language::Korean, "오늘 날씨가 좋습니다"),
            (Language::Spanish, "la reunión es a las tres."),
        ];
        for (language, input) in cases {
            let result = FillerFilter::for_language(language)
                .filter(input)
                .await
                .unwrap();
            assert_eq!(result.text, input, "{language:?} altered filler-free text");
        }
    }

    /// `for_language` must dispatch to the segmentation strategy that
    /// matches the script, not merely to some filter.
    #[test]
    fn for_language_picks_the_matching_segmentation() {
        assert!(matches!(
            FillerFilter::for_language(Language::Japanese),
            FillerFilter::Japanese(_)
        ));
        assert!(matches!(
            FillerFilter::for_language(Language::Chinese),
            FillerFilter::Chinese(_)
        ));
        assert!(matches!(
            FillerFilter::for_language(Language::Spanish),
            FillerFilter::Spanish(_)
        ));
        assert!(matches!(
            FillerFilter::for_language(Language::English),
            FillerFilter::Word(_)
        ));
        assert!(matches!(
            FillerFilter::for_language(Language::Korean),
            FillerFilter::Word(_)
        ));
    }

    #[test]
    fn bcp47_round_trips_and_rejects_unsupported() {
        for language in [
            Language::English,
            Language::Japanese,
            Language::Chinese,
            Language::Korean,
            Language::Spanish,
        ] {
            assert_eq!(Language::from_bcp47(language.as_bcp47()), Some(language));
        }
        // Region and script subtags are ignored, case is not significant.
        assert_eq!(Language::from_bcp47("en-US"), Some(Language::English));
        assert_eq!(Language::from_bcp47("zh-Hans"), Some(Language::Chinese));
        assert_eq!(Language::from_bcp47("JA"), Some(Language::Japanese));
        // Unsupported languages get no filter rather than a wrong one.
        assert_eq!(Language::from_bcp47("de"), None);
        assert_eq!(Language::from_bcp47(""), None);
    }
}