polyc_agent/

identifiers.rs

//! Normative identifier extraction for compaction retention (INV-C2).
//!
//! Anchored-iterative compaction replaces the conversation's anchor summary
//! wholesale, so an identifier the summary drops is dropped forever. The
//! retention invariant (INV-C2, conformance row CONF-2) is stated over
//! *extractable identifiers* — and this module is the single, normative
//! definition of that term. The compaction recall eval (#1134) and the
//! mint-time retention gate (#1135) both call this extractor; neither may
//! ship a private variant, or the eval would measure a different property
//! than the gate enforces.
//!
//! # Token classes
//!
//! An extractable identifier is a token (or token run) in one of five
//! classes, matching the conformance suite's ambiguity resolution:
//!
//! - [`IdentifierClass::Url`] — an `http://` or `https://` token, trailing
//!   punctuation trimmed.
//! - [`IdentifierClass::Amount`] — a numeral with a currency symbol
//!   (`$`/`€`/`£`), a trailing `%`, a following unit word from
//!   [`UNIT_WORDS`], or at least four digits (order numbers, PINs).
//! - [`IdentifierClass::OpaqueId`] — a machine-shaped token: at least five
//!   characters drawn from `[A-Za-z0-9._/-]` containing both a letter and a
//!   digit (UUIDs, `ord_93k2f7x`, `ZK-4471-BQ`).
//! - [`IdentifierClass::ProperNoun`] — a run of two or more consecutive
//!   capitalized words, leading English function words stripped
//!   (`Mirela Okafor`, not `The Mirela`).
//! - [`IdentifierClass::Quoted`] — the content of a straight double-quoted
//!   span of 3–120 bytes containing a letter. This is how an open question
//!   stays trackable: prose that tags it (`open item: "night berthing at
//!   dock 7"`) makes the tag extractable, while free interrogative prose is
//!   semantic content the recall eval measures but the gate cannot extract.
//!
//! # Survival predicate
//!
//! An identifier *survives* into a candidate text when the candidate
//! contains its text verbatim (case-sensitive substring —
//! [`is_retained`]). A paraphrase (`Okafor, Mirela`) does not count: the
//! gate's failure mode on over-strictness is rejecting a candidate summary
//! and keeping the prior anchor, which is the safe direction (INV-C3).

use std::collections::BTreeSet;

/// The class of an extracted identifier. See the module docs for the
/// normative grammar of each class.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum IdentifierClass {
    /// An `http://`/`https://` URL.
    Url,
    /// A numeral with currency, percent, a unit word, or ≥ 4 digits.
    Amount,
    /// A machine-shaped token mixing letters and digits.
    OpaqueId,
    /// A run of two or more capitalized words.
    ProperNoun,
    /// The content of a straight double-quoted span.
    Quoted,
}

/// One extracted identifier: its class and its verbatim text.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Identifier {
    /// Which token class matched.
    pub class: IdentifierClass,
    /// The verbatim text, exactly as it must reappear to survive.
    pub text: String,
}

/// Unit words that promote a bare numeral to an [`IdentifierClass::Amount`].
///
/// `512.4 kg` extracts because `kg` is listed here. The list is
/// deliberately small and closed: growing it loosens the gate for every
/// deployment at once, so additions belong in a reviewed change, not
/// configuration.
pub const UNIT_WORDS: &[&str] = &[
    "kg", "g", "t", "km", "m", "cm", "mm", "mi", "lb", "oz", "ms", "s", "min", "h", "GB", "MB",
    "TB", "KiB", "MiB", "kWh", "EUR", "USD", "GBP", "PLN",
];

/// English function words stripped from the head of a capitalized run so
/// sentence-initial `The Fenwick Boathouse` extracts as `Fenwick Boathouse`.
///
/// No single-letter entries (`A`, `I`): [`is_capitalized_word`] requires
/// `len() >= 2`, so a one-letter token never enters a run for this list to
/// strip.
const LEADING_STOPWORDS: &[&str] = &[
    "The", "An", "And", "But", "Or", "So", "If", "When", "While", "Then", "We", "It", "He", "She",
    "They", "You", "On", "In", "At", "To", "For", "From", "By", "With", "As", "Is", "Are", "Was",
    "Were", "Our", "My", "Your", "Their", "His", "Her", "No", "Not", "Yes", "Still", "Also",
    "Both", "Each", "This", "That", "These", "Those",
];

/// Extract every identifier in `text`, deduplicated and in a stable
/// (class, text) order.
///
/// This is the normative extraction function behind INV-C2: the recall
/// eval seeds transcripts with identifiers this function recognizes, and
/// the retention gate compares `extract(prior anchor)` against a candidate
/// summary. Determinism matters more than linguistic subtlety here — the
/// grammar is a closed, reviewable token scan, not a model call.
///
/// JSON/bracket structure (`"`, `{`, `}`, `[`, `]`) is treated as token
/// whitespace for the token-class scans, so an identifier embedded in an
/// unspaced machine payload (`"node":"node_j4x9q2"`) extracts exactly as it
/// would in prose. The quoted class reads the original text, where the
/// quotes still exist.
#[must_use]
pub fn extract_identifiers(text: &str) -> Vec<Identifier> {
    let mut out: BTreeSet<Identifier> = BTreeSet::new();
    let neutralized = text.replace(['"', '{', '}', '[', ']'], " ");
    extract_urls(&neutralized, &mut out);
    extract_quoted(text, &mut out);
    extract_token_classes(&neutralized, &mut out);
    extract_proper_nouns(&neutralized, &mut out);
    out.into_iter().collect()
}

/// The identifiers extracted from `prior` that do NOT survive (per
/// [`is_retained`]) into `candidate`.
///
/// This is the retention gate's core comparison (CONF-3, #1135): a
/// non-empty return means the candidate summary loses information the
/// prior anchor carried, and the candidate must be rejected with the prior
/// anchor kept authoritative.
#[must_use]
pub fn missing_identifiers(prior: &str, candidate: &str) -> Vec<Identifier> {
    extract_identifiers(prior)
        .into_iter()
        .filter(|id| !is_retained(candidate, &id.text))
        .collect()
}

/// The normative survival predicate: `candidate` retains `identifier` when
/// it contains the identifier's text verbatim (case-sensitive substring).
#[must_use]
pub fn is_retained(candidate: &str, identifier: &str) -> bool {
    candidate.contains(identifier)
}

/// Scan for `http://`/`https://` spans; capture to whitespace, trimming
/// trailing punctuation that prose or JSON quoting attaches.
fn extract_urls(text: &str, out: &mut BTreeSet<Identifier>) {
    for scheme in ["https://", "http://"] {
        let mut rest = text;
        while let Some(pos) = rest.find(scheme) {
            let tail = &rest[pos..];
            let end = tail.find(char::is_whitespace).unwrap_or(tail.len());
            let url = tail[..end].trim_end_matches(['.', ',', ';', ':', '!', '?', ')', '"', '\'']);
            if url.len() > scheme.len() {
                out.insert(Identifier {
                    class: IdentifierClass::Url,
                    text: url.to_owned(),
                });
            }
            rest = &tail[end.min(tail.len())..];
        }
    }
}

/// Scan for straight double-quoted spans of 3–120 bytes containing a letter
/// and no newline. Splitting on `"` puts quoted content at odd indices; an
/// odd-indexed FINAL segment has no closing quote and is skipped.
fn extract_quoted(text: &str, out: &mut BTreeSet<Identifier>) {
    let segments: Vec<&str> = text.split('"').collect();
    for (i, content) in segments.iter().enumerate().skip(1).step_by(2) {
        if i + 1 < segments.len()
            && (3..=120).contains(&content.len())
            && !content.contains('\n')
            && content.chars().any(|c| c.is_ascii_alphabetic())
        {
            out.insert(Identifier {
                class: IdentifierClass::Quoted,
                text: (*content).to_owned(),
            });
        }
    }
}

/// Trim prose punctuation from a whitespace token, keeping interior
/// symbols. Leading: everything before the first `[A-Za-z0-9$€£]`.
/// Trailing: everything after the last `[A-Za-z0-9%]`.
fn trim_token(token: &str) -> &str {
    let start = token
        .char_indices()
        .find(|(_, c)| c.is_ascii_alphanumeric() || matches!(c, '$' | '€' | '£'))
        .map(|(i, _)| i);
    let Some(start) = start else { return "" };
    let end = token
        .char_indices()
        .rev()
        .find(|(_, c)| c.is_ascii_alphanumeric() || *c == '%')
        .map(|(i, c)| i + c.len_utf8());
    let Some(end) = end else { return "" };
    if end <= start { "" } else { &token[start..end] }
}

/// True when `s` is `digits ( ',' digits )* ( '.' digits )?` — the numeral
/// core of the amount grammar.
fn is_numeral(s: &str) -> bool {
    if s.is_empty()
        || !s
            .chars()
            .all(|c| c.is_ascii_digit() || c == ',' || c == '.')
    {
        return false;
    }
    let mut chars = s.chars().peekable();
    if !chars.peek().is_some_and(char::is_ascii_digit) {
        return false;
    }
    let mut prev_sep = false;
    let mut seen_dot = false;
    for c in s.chars() {
        match c {
            ',' | '.' => {
                if prev_sep || (c == ',' && seen_dot) {
                    return false;
                }
                if c == '.' {
                    if seen_dot {
                        return false;
                    }
                    seen_dot = true;
                }
                prev_sep = true;
            }
            _ => prev_sep = false,
        }
    }
    !prev_sep
}

/// Per-token classes: amounts and opaque ids, with one-token lookahead for
/// unit words.
fn extract_token_classes(text: &str, out: &mut BTreeSet<Identifier>) {
    let tokens: Vec<&str> = text.split_whitespace().collect();
    for (i, raw) in tokens.iter().enumerate() {
        let tok = trim_token(raw);
        if tok.is_empty() {
            continue;
        }
        // Currency-prefixed numeral: `$12,845.03`, `€2,190`.
        if let Some(rest) = tok
            .strip_prefix('$')
            .or_else(|| tok.strip_prefix('€'))
            .or_else(|| tok.strip_prefix('£'))
        {
            if is_numeral(rest) {
                out.insert(Identifier {
                    class: IdentifierClass::Amount,
                    text: tok.to_owned(),
                });
            }
            continue;
        }
        // Percent: `85%`.
        if let Some(rest) = tok.strip_suffix('%') {
            if is_numeral(rest) {
                out.insert(Identifier {
                    class: IdentifierClass::Amount,
                    text: tok.to_owned(),
                });
            }
            continue;
        }
        if is_numeral(tok) {
            // Unit-followed numeral: `512.4 kg`.
            let unit = tokens.get(i + 1).map(|t| trim_token(t));
            if let Some(unit) = unit.filter(|u| UNIT_WORDS.contains(u)) {
                out.insert(Identifier {
                    class: IdentifierClass::Amount,
                    text: format!("{tok} {unit}"),
                });
                continue;
            }
            // Bare numeral with at least four digits: order numbers, PINs.
            if tok.chars().filter(char::is_ascii_digit).count() >= 4 {
                out.insert(Identifier {
                    class: IdentifierClass::Amount,
                    text: tok.to_owned(),
                });
            }
            continue;
        }
        // Opaque id: ≥ 5 chars of [A-Za-z0-9._/-] mixing letters and digits.
        if tok.len() >= 5
            && tok
                .chars()
                .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '/' | '-'))
            && tok.chars().any(|c| c.is_ascii_alphabetic())
            && tok.chars().any(|c| c.is_ascii_digit())
        {
            out.insert(Identifier {
                class: IdentifierClass::OpaqueId,
                text: tok.to_owned(),
            });
        }
    }
}

/// True for a capitalized word: `[A-Z]` then one or more `[a-z]`.
fn is_capitalized_word(s: &str) -> bool {
    let mut chars = s.chars();
    chars.next().is_some_and(|c| c.is_ascii_uppercase())
        && s.len() >= 2
        && chars.all(|c| c.is_ascii_lowercase())
}

/// Runs of ≥ 2 capitalized words, leading stopwords stripped. A run ends at
/// a token whose raw form carries trailing punctuation (`Ilves.` closes the
/// run), so names never join across a sentence or clause boundary.
fn extract_proper_nouns(text: &str, out: &mut BTreeSet<Identifier>) {
    let raw_tokens: Vec<&str> = text.split_whitespace().collect();
    let mut run: Vec<&str> = Vec::new();
    let mut flush = |run: &mut Vec<&str>| {
        let mut slice = run.as_slice();
        while let Some((head, rest)) = slice.split_first() {
            if LEADING_STOPWORDS.contains(head) {
                slice = rest;
            } else {
                break;
            }
        }
        if slice.len() >= 2 {
            out.insert(Identifier {
                class: IdentifierClass::ProperNoun,
                text: slice.join(" "),
            });
        }
        run.clear();
    };
    for raw in raw_tokens {
        let tok = trim_token(raw);
        if is_capitalized_word(tok) {
            run.push(tok);
            // Trailing punctuation on the raw token closes the clause — and
            // with it the run.
            if raw.ends_with(['.', ',', ';', ':', '!', '?', ')', '"', '\'']) {
                flush(&mut run);
            }
        } else {
            flush(&mut run);
        }
    }
    flush(&mut run);
}

#[cfg(test)]
mod tests {
    #![allow(clippy::pedantic, clippy::nursery, missing_docs)]

    use super::*;

    fn texts(ids: &[Identifier]) -> Vec<&str> {
        ids.iter().map(|i| i.text.as_str()).collect()
    }

    fn class_of(ids: &[Identifier], text: &str) -> Option<IdentifierClass> {
        ids.iter().find(|i| i.text == text).map(|i| i.class)
    }

    #[test]
    fn extracts_every_class_from_mixed_prose() {
        let text = "Our broker is Mirela Okafor; she filed entry ZK-4471-BQ for order ord_93k2f7x; \
                    duty came to $12,845.03 plus a 512.4 kg pallet. Manifest at \
                    https://port.example/manifests/BX-201. Open item: \"night berthing at dock 7\".";
        let ids = extract_identifiers(text);
        assert_eq!(
            class_of(&ids, "Mirela Okafor"),
            Some(IdentifierClass::ProperNoun)
        );
        assert_eq!(
            class_of(&ids, "ZK-4471-BQ"),
            Some(IdentifierClass::OpaqueId)
        );
        assert_eq!(
            class_of(&ids, "ord_93k2f7x"),
            Some(IdentifierClass::OpaqueId)
        );
        assert_eq!(class_of(&ids, "$12,845.03"), Some(IdentifierClass::Amount));
        assert_eq!(class_of(&ids, "512.4 kg"), Some(IdentifierClass::Amount));
        assert_eq!(
            class_of(&ids, "https://port.example/manifests/BX-201"),
            Some(IdentifierClass::Url)
        );
        assert_eq!(
            class_of(&ids, "night berthing at dock 7"),
            Some(IdentifierClass::Quoted)
        );
    }

    #[test]
    fn currency_and_percent_and_bare_numerals() {
        let ids = extract_identifiers("€2,190 due; retries at 85%; PIN 88417 set; row 212 done");
        assert_eq!(class_of(&ids, "€2,190"), Some(IdentifierClass::Amount));
        assert_eq!(class_of(&ids, "85%"), Some(IdentifierClass::Amount));
        assert_eq!(class_of(&ids, "88417"), Some(IdentifierClass::Amount));
        // A short bare numeral (three digits, no unit) is NOT an identifier —
        // gating on every small count would reject nearly any summary.
        assert!(!texts(&ids).contains(&"212"));
    }

    #[test]
    fn dates_and_plain_words_are_not_opaque_ids() {
        let ids = extract_identifiers("shipped 2026-07-16 with care by the harbor team");
        assert!(
            ids.is_empty(),
            "no letters+digits token, no ≥2-cap run, nothing quoted: {ids:?}"
        );
    }

    #[test]
    fn leading_stopword_is_stripped_from_proper_noun_runs() {
        let ids = extract_identifiers("The Fenwick Boathouse holds the booking.");
        assert_eq!(
            texts(&ids),
            vec!["Fenwick Boathouse"],
            "stopword stripped, run kept"
        );
        // A run that is ONLY a stopword plus one word still extracts the tail
        // pair only when two non-stopword words remain.
        let ids = extract_identifiers("The Boathouse holds the booking.");
        assert!(ids.is_empty(), "one non-stopword capitalized word is prose");
    }

    #[test]
    fn sentence_boundary_closes_a_proper_noun_run() {
        let ids = extract_identifiers("the coordinator is Tomas Ilves. Route it through him.");
        assert_eq!(
            texts(&ids),
            vec!["Tomas Ilves"],
            "trailing punctuation ends the run; the next sentence's opener is prose"
        );
    }

    #[test]
    fn quoted_spans_bound_length_and_need_a_letter() {
        let ids = extract_identifiers(r#"tagged "parking for the string quartet" and "12" and """#);
        assert_eq!(texts(&ids), vec!["parking for the string quartet"]);
    }

    #[test]
    fn urls_trim_trailing_prose_punctuation() {
        let ids = extract_identifiers("see https://tracker.example/c/7781, then reply");
        assert!(texts(&ids).contains(&"https://tracker.example/c/7781"));
    }

    #[test]
    fn uuids_extract_as_opaque_ids() {
        let ids = extract_identifiers("container 7f3d9a12-58c4-4de1-9b02-aa1c40f6d2e9 pinged");
        assert_eq!(
            class_of(&ids, "7f3d9a12-58c4-4de1-9b02-aa1c40f6d2e9"),
            Some(IdentifierClass::OpaqueId)
        );
    }

    #[test]
    fn json_embedded_identifiers_extract_like_prose() {
        // An unspaced machine payload: JSON syntax is token whitespace for
        // the token-class scans, so payload-only identifiers are extractable
        // (the tool-noise-heavy golden family depends on this).
        let ids = extract_identifiers(r#"{"node":"node_j4x9q2","cost":"$7,412.88"}"#);
        assert_eq!(
            class_of(&ids, "node_j4x9q2"),
            Some(IdentifierClass::OpaqueId)
        );
        assert_eq!(class_of(&ids, "$7,412.88"), Some(IdentifierClass::Amount));
    }

    #[test]
    fn missing_identifiers_flags_dropped_and_passes_retained() {
        let prior = "Entry ZK-4471-BQ cleared for Mirela Okafor at $12,845.03.";
        let keeps = "Customs entry ZK-4471-BQ (broker Mirela Okafor) settled: $12,845.03.";
        assert!(missing_identifiers(prior, keeps).is_empty());
        let drops = "Customs entry cleared for the broker; duty settled.";
        let missing = missing_identifiers(prior, drops);
        let missing_texts = texts(&missing);
        assert!(missing_texts.contains(&"ZK-4471-BQ"));
        assert!(missing_texts.contains(&"Mirela Okafor"));
        assert!(missing_texts.contains(&"$12,845.03"));
    }

    #[test]
    fn survival_is_verbatim_not_paraphrase() {
        assert!(is_retained("broker Mirela Okafor signed", "Mirela Okafor"));
        assert!(!is_retained(
            "broker Okafor, Mirela signed",
            "Mirela Okafor"
        ));
    }
}