vector_core/community/policy/
matchers.rs

1//! Text and link matching (§6) — the deterministic rules.
2//!
3//! Hit counting, shared by every text rule: patterns compile into one set,
4//! matching is leftmost-first and non-overlapping, and ONE hit is one distinct
5//! matched span (ten patterns hitting one span is one hit). Patterns sort
6//! bytewise before matching so their declaration order cannot decide which span
7//! wins.
8
9use super::document::{ExemptKind, ExemptPatterns};
10use super::normalize::is_token_char;
11use super::types::Span;
12
13/// One matched span in the normalized text.
14#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
15pub struct Hit {
16    pub start: usize,
17    pub end: usize,
18}
19
20impl Hit {
21    pub fn span(self) -> Span {
22        Span { start: self.start as u32, end: self.end as u32 }
23    }
24}
25
26/// A keyword pattern's anchoring, from the Discord-compatible grammar:
27/// bare = token-anchored (the Scunthorpe guard), `*w` / `w*` / `*w*` relax an
28/// edge. `\*` is a literal asterisk.
29#[derive(Debug, Clone, PartialEq, Eq)]
30struct Keyword {
31    needle: String,
32    open_left: bool,
33    open_right: bool,
34}
35
36fn parse_keyword(pattern: &str) -> Keyword {
37    let mut s = pattern;
38    let mut open_left = false;
39    let mut open_right = false;
40    if let Some(rest) = s.strip_prefix('*') {
41        open_left = true;
42        s = rest;
43    }
44    if s.ends_with('*') && !s.ends_with("\\*") {
45        open_right = true;
46        s = &s[..s.len() - 1];
47    }
48    Keyword { needle: s.replace("\\*", "*"), open_left, open_right }
49}
50
51/// Find every non-overlapping hit for a set of keyword patterns in already
52/// normalized text. Patterns are sorted bytewise first; at one start position
53/// the lowest sorted index wins.
54pub fn keyword_hits(text: &str, patterns: &[String]) -> Vec<Hit> {
55    let mut kws: Vec<Keyword> = patterns.iter().map(|p| parse_keyword(p)).filter(|k| !k.needle.is_empty()).collect();
56    kws.sort_by(|a, b| a.needle.cmp(&b.needle));
57    let bytes = text.as_bytes();
58    let mut hits: Vec<Hit> = Vec::new();
59    let mut cursor = 0usize;
60    while cursor < text.len() {
61        let mut best: Option<Hit> = None;
62        for k in &kws {
63            let mut from = cursor;
64            while let Some(rel) = text[from..].find(&k.needle) {
65                let start = from + rel;
66                let end = start + k.needle.len();
67                let left_ok = k.open_left || start == 0 || !is_token_char(prev_char(text, start));
68                let right_ok = k.open_right || end == text.len() || !is_token_char(next_char(text, end));
69                if left_ok && right_ok {
70                    // Leftmost wins; at equal starts the longer span, then the
71                    // lower sorted index (already the iteration order).
72                    let candidate = Hit { start, end };
73                    best = match best {
74                        Some(b) if (b.start, std::cmp::Reverse(b.end)) <= (candidate.start, std::cmp::Reverse(candidate.end)) => Some(b),
75                        _ => Some(candidate),
76                    };
77                    break;
78                }
79                let step = next_char_boundary(bytes, start);
80                if step <= from {
81                    break;
82                }
83                from = step;
84            }
85        }
86        match best {
87            Some(h) => {
88                cursor = h.end.max(next_char_boundary(bytes, h.start));
89                hits.push(h);
90            }
91            None => break,
92        }
93    }
94    hits.sort();
95    hits
96}
97
98fn prev_char(text: &str, at: usize) -> char {
99    text[..at].chars().next_back().unwrap_or(' ')
100}
101
102fn next_char(text: &str, at: usize) -> char {
103    text[at..].chars().next().unwrap_or(' ')
104}
105
106fn next_char_boundary(bytes: &[u8], from: usize) -> usize {
107    let mut i = from + 1;
108    while i < bytes.len() && (bytes[i] & 0xC0) == 0x80 {
109        i += 1;
110    }
111    i
112}
113
114/// Cancel hits an exemption covers: a hit is cancelled when an exempt match
115/// CONTAINS or EQUALS its span. Exemptions match under the citing rule's
116/// normalizer, so both live in one coordinate space.
117pub fn cancel_exempt_hits(text: &str, hits: Vec<Hit>, exempts: &[&ExemptPatterns]) -> Vec<Hit> {
118    let mut covers: Vec<Hit> = Vec::new();
119    for e in exempts {
120        match e.kind {
121            Some(ExemptKind::Literal) | None => {
122                for v in &e.values {
123                    let mut from = 0usize;
124                    while let Some(rel) = text[from..].find(v.as_str()) {
125                        let start = from + rel;
126                        covers.push(Hit { start, end: start + v.len() });
127                        from = start + v.len().max(1);
128                    }
129                }
130            }
131            Some(ExemptKind::Wildcard) => {
132                for v in &e.values {
133                    covers.extend(keyword_hits(text, std::slice::from_ref(v)));
134                }
135            }
136            // Domain exemptions belong to `link`, which matches in the raw
137            // registrable-domain space, not here.
138            Some(ExemptKind::Domain) => {}
139        }
140    }
141    hits.into_iter().filter(|h| !covers.iter().any(|c| c.start <= h.start && c.end >= h.end)).collect()
142}
143
144// ── Links (§7.4 extraction grammar; raw registrable-domain space) ────────────
145
146/// Extract the registrable domains a message links to. Absolute URLs need an
147/// explicit http(s) scheme; a bare host must look like `label(.label)+`.
148/// Trailing punctuation is stripped, userinfo resolves as host (not path), and
149/// IDN maps to punycode BEFORE anchoring.
150pub fn extract_domains(text: &str) -> Vec<String> {
151    let mut out: Vec<String> = Vec::new();
152    for raw in text.split(|c: char| c.is_whitespace() || c == '<' || c == '>' || c == '"' || c == '(' || c == ')') {
153        let token = raw.trim_end_matches(|c| ".,;:!?)]}'\"".contains(c));
154        if token.is_empty() {
155            continue;
156        }
157        let lowered = token.to_lowercase();
158        let after_scheme = match lowered.split_once("://") {
159            Some((scheme, rest)) => {
160                if scheme != "http" && scheme != "https" {
161                    continue;
162                }
163                rest
164            }
165            None => lowered.as_str(),
166        };
167        // Host ends at the first path/query/fragment separator; userinfo before
168        // '@' is discarded so `evil.com/x@good.io` cannot masquerade.
169        let hostport = after_scheme.split(['/', '?', '#']).next().unwrap_or("");
170        let host = hostport.rsplit_once('@').map(|(_, h)| h).unwrap_or(hostport);
171        let host = host.split(':').next().unwrap_or("").trim_end_matches('.');
172        if host.is_empty() || !host.contains('.') {
173            continue;
174        }
175        if !host.chars().all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-') {
176            continue;
177        }
178        let d = registrable_domain(host);
179        if !d.is_empty() && !out.contains(&d) {
180            out.push(d);
181        }
182    }
183    out
184}
185
186/// The registrable domain, anchored so `vectorapp.io.evil.com` and
187/// `evil.com/vectorapp.io` can never pass an allowlist.
188///
189/// Bundle v1 ships a small public-suffix set: the full PSL snapshot is a
190/// ratification artifact, so a multi-label suffix outside this set falls back
191/// to the last two labels. That is the one approximation in the matcher, and it
192/// is conservative — it never merges two distinct registrable domains.
193fn registrable_domain(host: &str) -> String {
194    const MULTI: &[&str] = &[
195        "co.uk", "org.uk", "ac.uk", "gov.uk", "co.jp", "or.jp", "ne.jp", "com.au", "net.au", "org.au", "co.nz",
196        "com.br", "com.cn", "com.mx", "co.za", "co.in", "co.kr", "github.io", "gitlab.io", "pages.dev",
197        "workers.dev", "vercel.app", "netlify.app", "herokuapp.com", "s3.amazonaws.com",
198    ];
199    let labels: Vec<&str> = host.split('.').collect();
200    if labels.len() < 2 {
201        return host.to_string();
202    }
203    let last2 = labels[labels.len() - 2..].join(".");
204    if MULTI.contains(&last2.as_str()) && labels.len() >= 3 {
205        return labels[labels.len() - 3..].join(".");
206    }
207    last2
208}
209
210/// Non-exempt domains in a message: the `link` rule's hits.
211pub fn link_hits(text: &str, patterns: &[String], exempts: &[&ExemptPatterns]) -> Vec<String> {
212    let allow: Vec<String> = exempts
213        .iter()
214        .filter(|e| matches!(e.kind, Some(ExemptKind::Domain) | Some(ExemptKind::Literal) | None))
215        .flat_map(|e| e.values.iter().map(|v| registrable_domain(&v.to_lowercase())))
216        .collect();
217    extract_domains(text)
218        .into_iter()
219        .filter(|d| {
220            // Cancellation is by EQUALITY on the hit's own value: these hits
221            // carry no span, so substring containment has no meaning here.
222            if allow.iter().any(|a| a == d) {
223                return false;
224            }
225            patterns.is_empty() || patterns.iter().any(|p| registrable_domain(&p.to_lowercase()) == *d)
226        })
227        .collect()
228}
229
230/// How many distinct people a message named.
231///
232/// Vector has no mention TAG: a mention is an npub written in the body, which the
233/// renderer turns into a pill. The only `p` tags a community message carries are
234/// its reply parent and reaction target, so counting those scored 1 on an
235/// ordinary reply and 0 on a message that named fifteen people.
236///
237/// Grammar comes from [`crate::types::extract_mentions`] — the same scanner the
238/// rest of the app already trusts, rather than a second opinion on what an npub
239/// looks like. Distinct people, not occurrences: naming someone three times is
240/// still one person called, and a rule about mass-pinging cares how many were
241/// pulled in.
242pub fn count_mentions(text: &str) -> u32 {
243    let mut seen: Vec<&str> = Vec::new();
244    for npub in crate::types::extract_mentions(text) {
245        if seen.contains(&npub) {
246            continue;
247        }
248        // One npub can appear twice: linked once and called once. Being linked
249        // anywhere does not cancel being called somewhere else.
250        if text.match_indices(npub).any(|(at, _)| !is_linked(text, at)) {
251            seen.push(npub);
252        }
253    }
254    seen.len() as u32
255}
256
257/// Whether the npub at `at` is part of a URL rather than someone being called.
258///
259/// `extract_mentions` rejects only an ALPHANUMERIC neighbour, which leaves
260/// `…/profile/npub1…` looking like a mention. Linking to a profile is not
261/// pinging its owner, so the policy engine applies the frontend's wider guard
262/// here rather than tightening a shared helper other callers rely on.
263fn is_linked(text: &str, at: usize) -> bool {
264    let bytes = text.as_bytes();
265    let mut lead = at;
266    if lead >= 1 && bytes[lead - 1] == b'@' {
267        lead -= 1;
268    } else if lead >= 6 && text.get(lead - 6..lead).is_some_and(|p| p.eq_ignore_ascii_case("nostr:")) {
269        lead -= 6;
270    }
271    if lead == 0 {
272        return false;
273    }
274    let prev = text[..lead].chars().next_back().unwrap_or(' ');
275    matches!(prev, '/' | '=' | '?' | '&' | '#' | '%' | '.' | '-' | '_')
276}
277
278#[cfg(test)]
279mod tests {
280    use super::*;
281
282    fn pats(v: &[&str]) -> Vec<String> {
283        v.iter().map(|s| s.to_string()).collect()
284    }
285
286    #[test]
287    fn bare_patterns_are_token_anchored() {
288        // The Scunthorpe guard: a bare word never matches inside another.
289        assert_eq!(keyword_hits("classic assassin", &pats(&["ass"])).len(), 0);
290        assert_eq!(keyword_hits("what an ass", &pats(&["ass"])).len(), 1);
291        // Separators bound a token; digits continue it.
292        assert_eq!(keyword_hits("beta. beta_x", &pats(&["beta"])).len(), 2);
293        assert_eq!(keyword_hits("beta2", &pats(&["beta"])).len(), 0);
294    }
295
296    #[test]
297    fn wildcards_relax_the_edges() {
298        assert_eq!(keyword_hits("scammer", &pats(&["scam*"])).len(), 1);
299        assert_eq!(keyword_hits("scammer", &pats(&["*scam"])).len(), 0, "left-open still anchors the right");
300        assert_eq!(keyword_hits("descammer", &pats(&["*scam*"])).len(), 1);
301        // An escaped asterisk is a literal.
302        assert_eq!(keyword_hits("buy * now", &pats(&["\\*"])).len(), 1);
303    }
304
305    #[test]
306    fn one_span_is_one_hit_regardless_of_pattern_count() {
307        let hits = keyword_hits("free airdrop free airdrop", &pats(&["free", "airdrop"]));
308        assert_eq!(hits.len(), 4, "four distinct spans");
309        // Overlapping patterns over one span collapse to a single hit.
310        let hits = keyword_hits("*scam*", &pats(&["*scam*", "*sca*", "*cam*"]));
311        assert_eq!(hits.len(), 1);
312    }
313
314    #[test]
315    fn declaration_order_never_decides_a_span() {
316        let a = keyword_hits("free airdrop", &pats(&["free", "airdrop"]));
317        let b = keyword_hits("free airdrop", &pats(&["airdrop", "free"]));
318        assert_eq!(a, b);
319    }
320
321    #[test]
322    fn exemptions_cancel_by_containment() {
323        let text = "join the free airdrop channel";
324        let hits = keyword_hits(text, &pats(&["airdrop"]));
325        assert_eq!(hits.len(), 1);
326        let ex = ExemptPatterns { kind: Some(ExemptKind::Literal), values: pats(&["free airdrop"]) };
327        assert!(cancel_exempt_hits(text, hits, &[&ex]).is_empty(), "an exempt phrase covers the hit inside it");
328    }
329
330    #[test]
331    fn domains_anchor_on_the_registrable_name() {
332        assert_eq!(extract_domains("see https://vectorapp.io/x"), vec!["vectorapp.io"]);
333        // The oldest allowlist bypasses.
334        assert_eq!(extract_domains("https://vectorapp.io.evil.com/a"), vec!["evil.com"]);
335        assert_eq!(extract_domains("https://evil.com/vectorapp.io"), vec!["evil.com"]);
336        // Userinfo resolves as host, not path.
337        assert_eq!(extract_domains("https://vectorapp.io@evil.com/"), vec!["evil.com"]);
338        // Trailing punctuation and ports fall away; a bare host still counts.
339        assert_eq!(extract_domains("go to bit.ly:8080, now"), vec!["bit.ly"]);
340        // Multi-label suffixes keep their registrable label.
341        assert_eq!(extract_domains("https://foo.github.io/p"), vec!["foo.github.io"]);
342        assert_eq!(extract_domains("https://a.b.co.uk/p"), vec!["b.co.uk"]);
343        // Non-http schemes and bare words are not links.
344        assert!(extract_domains("ftp://x.com nothing here").is_empty());
345    }
346
347    #[test]
348    fn link_allowlists_cancel_by_equality_only() {
349        let allow = ExemptPatterns { kind: Some(ExemptKind::Domain), values: pats(&["vectorapp.io"]) };
350        assert!(link_hits("https://vectorapp.io/a", &[], &[&allow]).is_empty());
351        // Substring must never cancel: app.io does not exempt myapp.io.
352        let near = ExemptPatterns { kind: Some(ExemptKind::Domain), values: pats(&["app.io"]) };
353        assert_eq!(link_hits("https://myapp.io/a", &[], &[&near]), vec!["myapp.io"]);
354        // Distinct domains per message, deduped.
355        let hits = link_hits("a bit.ly/x and bit.ly/y and tr.ee/z", &[], &[]);
356        assert_eq!(hits, vec!["bit.ly", "tr.ee"]);
357    }
358
359    #[test]
360    fn a_denylist_matches_only_its_own_domains() {
361        let deny = pats(&["bit.ly", "tr.ee"]);
362        assert_eq!(link_hits("see bit.ly/x", &deny, &[]), vec!["bit.ly"]);
363        assert!(link_hits("see github.com/x", &deny, &[]).is_empty());
364    }
365
366    /// Built, not typed: a hand-counted 58-char literal is a test that fails for
367    /// the wrong reason.
368    fn npub_of(c: char) -> String {
369        format!("npub1{}", std::iter::repeat(c).take(58).collect::<String>())
370    }
371
372    #[test]
373    fn a_mention_is_an_npub_in_the_body_however_it_is_written() {
374        let a = npub_of('q');
375        assert_eq!(count_mentions(&format!("hey @{a} look")), 1);
376        assert_eq!(count_mentions(&format!("hey nostr:{a} look")), 1);
377        assert_eq!(count_mentions(&format!("hey {a} look")), 1);
378    }
379
380    #[test]
381    fn naming_one_person_twice_is_still_one_person() {
382        let a = npub_of('q');
383        let b = npub_of('z');
384        assert_eq!(count_mentions(&format!("@{a} and again @{a}")), 1);
385        assert_eq!(count_mentions(&format!("@{a} and @{b}")), 2);
386    }
387
388    /// The shape the rule exists for.
389    #[test]
390    fn a_mass_ping_counts_everyone_it_named() {
391        let a = npub_of('q');
392        let b = npub_of('z');
393        assert_eq!(count_mentions(&format!("@{a} @{b} @{a} ping")), 2);
394    }
395
396    /// A profile URL is not someone calling your name, and neither is an npub
397    /// welded to a word.
398    #[test]
399    fn an_npub_inside_a_url_or_a_word_is_not_a_mention() {
400        let a = npub_of('q');
401        assert_eq!(count_mentions(&format!("https://vectorapp.io/profile/{a}")), 0);
402        assert_eq!(count_mentions(&format!("x{a}")), 0);
403        assert_eq!(count_mentions(&format!("?p={a}")), 0);
404    }
405
406    #[test]
407    fn a_malformed_npub_is_not_a_mention() {
408        assert_eq!(count_mentions("npub1short"), 0);
409        // 59 data chars: a longer string that merely starts like an npub.
410        assert_eq!(count_mentions(&format!("{}q", npub_of('q'))), 0);
411    }
412
413    #[test]
414    fn plain_text_names_nobody() {
415        assert_eq!(count_mentions("just a normal message @everyone"), 0);
416    }
417}
vector_core/community/policy/matchers.rs

vector_core/community/policy/
matchers.rs
