disarm 0.14.0

Unicode canonicalization and TR39 visual confusable analysis: building blocks for text-security pipelines (homoglyph/bidi/zalgo handling) plus standards-based phonetic transliteration
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

//! Unicode TR39 confusable character mappings (multi-target).
//!
//! Auto-generated from confusables.txt by scripts/gen_confusables.py. The upstream
//! release is not restated here — [`CONFUSABLES_VERSION`] carries it, parsed from the
//! TSV header by build.rs so there is exactly one place it can be wrong (#560).
//!
//! Contains 2,220 non-Latin → Latin mappings and 1,349 non-Cyrillic →
//! Cyrillic mappings. Uses compile-time perfect hash maps (`phf`) for O(1)
//! lookups. Covers Cyrillic, Greek, Armenian, Georgian, CJK compatibility,
//! mathematical symbols, fullwidth forms, and other confusable characters.
//!
//! PHF maps generated by build.rs from src/tables/data/confusables_to_*.tsv.
//! To update the data, regenerate with: python scripts/gen_confusables.py

// Non-Latin → Latin confusable mappings.
include!(concat!(env!("OUT_DIR"), "/confusables_phf.rs"));

// ASCII codepoints the latin map rewrites (preset fast-path guard, #458).
include!(concat!(env!("OUT_DIR"), "/ascii_confusable_latin.rs"));

// Non-Cyrillic → Cyrillic confusable mappings.
include!(concat!(env!("OUT_DIR"), "/confusables_to_cyrillic_phf.rs"));

// The upstream `confusables.txt` release both tables were folded from (#560), parsed
// from the TSV header by build.rs rather than typed here a second time.
include!(concat!(env!("OUT_DIR"), "/confusables_version.rs"));

// Digit-policy overrides (#561): TR39's target for the rows where disarm diverges.
include!(concat!(env!("OUT_DIR"), "/confusables_digit_tr39_phf.rs"));

// Contraction rules (#562): ASCII digraph -> single ASCII letter, hostname path only.
include!(concat!(env!("OUT_DIR"), "/confusables_contractions.rs"));

// Every source codepoint in the upstream confusables.txt (#563). The coverage
// denominator — see `unmapped_sources`.
include!(concat!(
    env!("OUT_DIR"),
    "/confusables_upstream_sources_phf.rs"
));

/// TR39's target for `ch`, when disarm's own table diverges from it on digit policy
/// (#561). `None` means the two agree and the main table's value stands.
///
/// Consulted only under `digit_policy = "tr39"`; the default numeric policy never reaches
/// this map, so it costs nothing on the common path.
#[inline]
pub fn digit_tr39_override(ch: char) -> Option<&'static str> {
    DIGIT_TR39.get(&ch).copied()
}

/// True if `ch` is a confusable **source** in the bundled upstream `confusables.txt`,
/// whether or not disarm's tables fold it (#563).
///
/// Pair with [`lookup`] to answer the coverage question: a character that is an
/// upstream source but has no mapping for the chosen target is one disarm does not
/// neutralize — which is exactly where an adaptive attacker goes next.
#[inline]
pub fn is_upstream_source(ch: char) -> bool {
    UPSTREAM_CONFUSABLE_SOURCES.contains(&ch)
}

/// Every upstream confusable source the `target_script` table does not map, sorted.
///
/// Derived rather than stored: the answer is the upstream source set minus the
/// resolved table's keys, so it cannot drift from the table it describes, and one
/// generated denominator serves both targets. Returns an empty vec for an unknown
/// script (callers validate first).
pub fn unmapped_sources(target_script: &str) -> Vec<char> {
    let Some(map) = resolve_map(target_script) else {
        return Vec::new();
    };
    let mut out: Vec<char> = UPSTREAM_CONFUSABLE_SOURCES
        .iter()
        .copied()
        .filter(|ch| !map.contains_key(ch))
        .collect();
    // PHF iteration order is the hash order. Sort so the API is deterministic across
    // builds — a caller diffing two releases' exposure sets needs a stable order.
    out.sort_unstable();
    out
}

/// Look up a confusable mapping for a character to the target script.
///
/// Returns the target-script equivalent if the character is a known
/// confusable, or None if it is not.
///
/// Supported target scripts: `"latin"`, `"cyrillic"`.
#[inline]
pub fn lookup(ch: char, target_script: &str) -> Option<&'static str> {
    resolve_map(target_script).and_then(|m| m.get(&ch).copied())
}

/// Resolve a `target_script` to its confusables PHF map, once.
///
/// Lets callers hoist the `match target_script` out of a per-character loop
/// (#236 / #233 review item) and probe the resolved map directly. Returns
/// `None` for an unknown script.
///
/// Note: there is intentionally **no** ASCII fast path built on top of this —
/// the latin table maps ASCII source code points (e.g. U+007C `|`→`l`,
/// U+0022 `"`→`''`, U+0060 `` ` ``→`'`), so ASCII input is *not* identity even
/// for `target="latin"`.
#[inline]
pub fn resolve_map(target_script: &str) -> Option<&'static phf::Map<char, &'static str>> {
    match target_script {
        "latin" => Some(&TO_LATIN),
        "cyrillic" => Some(&TO_CYRILLIC),
        _ => None,
    }
}