use gaze::{
Action, ClassRule, CleanDocument, DefaultRule, PiiClass, Pipeline, RawDocument, Scope, Session,
};
use gaze_recognizers::{NormalizerKind, RegexDetector, ValidatorKind};
#[cfg(not(feature = "phone-parser"))]
use gaze_types::ValidatorKindParseError;
use sha3::{Digest, Sha3_256};
fn validator_pipeline(
pattern: &str,
validator: ValidatorKind,
normalizer: Option<NormalizerKind>,
) -> Pipeline {
custom_validator_pipeline(
pattern,
PiiClass::custom("credit_card_or_iban"),
validator,
normalizer,
)
}
fn custom_validator_pipeline(
pattern: &str,
class: PiiClass,
validator: ValidatorKind,
normalizer: Option<NormalizerKind>,
) -> Pipeline {
Pipeline::builder()
.recognizer(
RegexDetector::with_rulepack_fields(
pattern,
class.clone(),
"s1.validator.test",
vec![gaze::LocaleTag::Global],
0.90,
50,
"counter",
None,
Vec::new(),
Some(validator),
normalizer,
)
.expect("regex detector"),
)
.rule(ClassRule::new(class, Action::Tokenize))
.rule(DefaultRule::new(Action::Preserve))
.build()
.expect("pipeline")
}
fn clean_text(pipeline: &Pipeline, session: &Session, input: &str) -> String {
let clean = pipeline
.redact(session, RawDocument::Text(input.to_string()))
.expect("redact");
let CleanDocument::Text(text) = clean else {
panic!("expected text document");
};
text
}
fn restore_tokens(session: &Session, clean: &str) -> String {
gaze::token_shape::pattern()
.replace_all(clean, |captures: ®ex::Captures<'_>| {
session.restore_strict(&captures[0]).expect("known token")
})
.to_string()
}
#[test]
fn luhn_algorithm_accepts_public_test_numbers_and_rejects_mutants() {
for input in [
"4111111111111111",
"4012888888881881",
"5555555555554444",
"5105105105105100",
"378282246310005",
"371449635398431",
"4111 1111 1111 1111",
] {
assert!(ValidatorKind::Luhn.validates(input), "{input}");
}
for input in ["4111111111111112", "5555555555554445", "378282246310006"] {
assert!(!ValidatorKind::Luhn.validates(input), "{input}");
}
}
#[test]
fn iban_mod97_accepts_spec_examples_and_rejects_check_digit_mutants() {
for input in [
"NO93 8601 1117 947",
"NO9386011117947",
"BE71 0961 2345 6769",
"DE89 3704 0044 0532 0130 00",
"FR14 2004 1010 0505 0001 3M02 606",
"GB82 WEST 1234 5698 7654 32",
"MT84 MALT 0110 0001 2345 MTLC AST0 01S",
"LC14 BOSL 1234 5678 9012 3456 7890 1234",
] {
assert!(ValidatorKind::IbanMod97.validates(input), "{input}");
}
for input in ["GB99WEST12345698765432", "DE99370400440532013000"] {
assert!(!ValidatorKind::IbanMod97.validates(input), "{input}");
}
}
#[test]
fn iban_mod97_rejects_mod97_passing_wrong_country_lengths() {
for input in [
"NO37 8601 1117 9470",
"DE81 3704 0044 0532 0130 000",
"GB76 WEST 1234 5698 7654 320",
"BE30 0961 2345 6769 0",
"LC22 BOSL 1234 5678 9012 3456 7890 1234 0",
] {
assert!(!ValidatorKind::IbanMod97.validates(input), "{input}");
}
}
#[test]
fn ipv4_parse_accepts_rfc5737_documentation_addresses_and_rejects_malformed() {
for input in [
"192.0.2.1",
"198.51.100.10",
"203.0.113.255",
"0.0.0.0",
"255.255.255.255",
] {
assert!(ValidatorKind::Ipv4Parse.validates(input), "{input}");
}
for input in [
"192.000.2.1",
"01.2.3.4",
"0xC0.0x00.0x02.0x01",
"192.168.1",
"192.168",
"256.0.0.1",
"192.0.2.300",
"192.0.2.1.",
" 192.0.2.1",
"",
] {
assert!(!ValidatorKind::Ipv4Parse.validates(input), "{input}");
}
}
#[test]
fn ipv4_parse_kind_token_round_trips() {
let kind = ValidatorKind::parse("ipv4_parse").expect("parse ipv4_parse");
assert_eq!(kind, ValidatorKind::Ipv4Parse);
}
#[test]
fn ipv6_parse_accepts_rfc3849_documentation_addresses_and_ipv4_embedded() {
for input in [
"2001:db8::1",
"::1",
"::",
"fe80::1",
"2001:db8:0:0:0:0:0:1",
"::ffff:192.0.2.128",
"::192.0.2.1",
"2001:db8::192.0.2.1",
"fe80::ffff:1.2.3.4",
"0:0:0:0:0:ffff:192.0.2.1",
] {
assert!(ValidatorKind::Ipv6Parse.validates(input), "{input}");
}
for input in [
"2001::1::2",
"[2001:db8::1]",
"fe80::1%eth0",
"::ffff:256.0.0.1",
"",
" 2001:db8::1",
] {
assert!(!ValidatorKind::Ipv6Parse.validates(input), "{input}");
}
}
#[test]
fn ipv6_parse_kind_token_round_trips() {
let kind = ValidatorKind::parse("ipv6_parse").expect("parse ipv6_parse");
assert_eq!(kind, ValidatorKind::Ipv6Parse);
}
#[test]
fn eth_eip55_accepts_spec_vectors_and_rejects_mutants() {
for input in [
"0x52908400098527886E0F7030069857D2E4169EE7",
"0x8617E340B3D01FA5F11F306F4090FD50E238070D",
"0xde709f2102306220921060314715629080e2fb77",
"0x27b1fdb04752bbc536007a920d24acb045561c26",
"0x5aAeb6053F3E94C9b9A09f33669435E7Ef1BeAed",
"0xfB6916095ca1df60bB79Ce92cE3Ea74c37c5d359",
"0xdbF03B407c01E7cD3CBea99509d93f8DDDC8C6FB",
"0xD1220A0cf47c7B9Be7A2E6BA89F429762e7b9aDb",
] {
assert!(ValidatorKind::EthEip55.validates(input), "{input}");
}
for input in [
"0x52908400098527886e0F7030069857D2E4169EE7",
"0xfB6916095ca1df60bB79Ce92cE3Ea74c37c5d358",
"52908400098527886E0F7030069857D2E4169EE7",
"0x52908400098527886E0F7030069857D2E4169EE",
"0x52908400098527886E0F7030069857D2E4169EE7Z",
"",
] {
assert!(!ValidatorKind::EthEip55.validates(input), "{input}");
}
}
#[test]
fn eth_eip55_kind_token_round_trips() {
let kind = ValidatorKind::parse("eth_eip55").expect("parse eth_eip55");
assert_eq!(kind, ValidatorKind::EthEip55);
}
#[test]
fn eth_eip55_spec_vectors_fail_under_nist_sha3_256() {
for input in [
"0x5aAeb6053F3E94C9b9A09f33669435E7Ef1BeAed",
"0xfB6916095ca1df60bB79Ce92cE3Ea74c37c5d359",
"0xdbF03B407c01E7cD3CBea99509d93f8DDDC8C6FB",
"0xD1220A0cf47c7B9Be7A2E6BA89F429762e7b9aDb",
] {
assert!(!eth_eip55_check_with_nist_sha3_256(input), "{input}");
}
}
fn eth_eip55_check_with_nist_sha3_256(input: &str) -> bool {
let Some(address) = input.strip_prefix("0x") else {
return false;
};
if address.len() != 40 || !address.bytes().all(|byte| byte.is_ascii_hexdigit()) {
return false;
}
let lowercase = address.to_ascii_lowercase();
let hash = Sha3_256::digest(lowercase.as_bytes());
for (index, byte) in address.bytes().enumerate() {
if byte.is_ascii_digit() {
continue;
}
let hash_nibble = if index % 2 == 0 {
hash[index / 2] >> 4
} else {
hash[index / 2] & 0x0f
};
if (hash_nibble > 7) != byte.is_ascii_uppercase() {
return false;
}
}
true
}
#[cfg(not(feature = "phone-parser"))]
#[test]
fn e164_phone_fails_closed_when_phone_parser_feature_disabled() {
let result = ValidatorKind::parse("e164_phone");
assert!(
matches!(
result,
Err(ValidatorKindParseError::UnsupportedValidator { ref kind }) if kind == "e164_phone"
),
"feature-disabled build MUST reject e164_phone with UnsupportedValidator (axis-1 fail-closed); got: {result:?}"
);
}
#[cfg(feature = "phone-parser")]
#[test]
fn e164_phone_accepts_assigned_international_number_and_rejects_unassigned_prefix() {
assert!(ValidatorKind::E164Phone.validates("+4915550112233"));
assert!(!ValidatorKind::E164Phone.validates("+99999999"));
assert!(!ValidatorKind::E164Phone.validates("4915550112233"));
}
#[test]
fn iban_canonical_is_uppercase_whitespace_free_and_idempotent() {
let canonical = NormalizerKind::IbanCanonical.normalize("gb82 west 1234 5698 7654 32");
assert_eq!(canonical, "GB82WEST12345698765432");
assert_eq!(
NormalizerKind::IbanCanonical.normalize(&canonical),
canonical
);
}
#[cfg(feature = "phone-parser")]
#[test]
fn s3a_e164_phone_passing_candidate_emits_detection_and_round_trips() {
let pipeline = custom_validator_pipeline(
r"\+\d{6,15}\b",
PiiClass::custom("phone"),
ValidatorKind::E164Phone,
None,
);
let session = Session::new(Scope::Ephemeral).expect("session");
let input = "Phone: +4915550112233";
let clean = clean_text(&pipeline, &session, input);
assert!(clean.starts_with("Phone: <"), "{clean}");
assert!(clean.ends_with(":Custom:phone_1>"), "{clean}");
assert_eq!(restore_tokens(&session, &clean), input);
}
#[cfg(feature = "phone-parser")]
#[test]
fn s3a_e164_phone_unassigned_candidate_emits_no_detection() {
let pipeline = custom_validator_pipeline(
r"\+\d{6,15}\b",
PiiClass::custom("phone"),
ValidatorKind::E164Phone,
None,
);
let session = Session::new(Scope::Ephemeral).expect("session");
let input = "Phone: +99999999";
let clean = clean_text(&pipeline, &session, input);
assert_eq!(clean, input);
}
#[test]
fn s1_luhn_passing_card_emits_detection() {
let pipeline = validator_pipeline(r"\b\d{16}\b", ValidatorKind::Luhn, None);
let session = Session::new(Scope::Ephemeral).expect("session");
let input = "Card: 4111111111111111";
let clean = clean_text(&pipeline, &session, input);
assert!(clean.starts_with("Card: <"), "{clean}");
assert!(clean.ends_with(":Custom:credit_card_or_iban_1>"), "{clean}");
assert_eq!(restore_tokens(&session, &clean), input);
}
#[test]
fn s1_luhn_failing_card_emits_no_detection() {
let pipeline = validator_pipeline(r"\b\d{16}\b", ValidatorKind::Luhn, None);
let session = Session::new(Scope::Ephemeral).expect("session");
let input = "Card: 4111111111111112";
let clean = clean_text(&pipeline, &session, input);
assert_eq!(clean, input);
}
#[test]
fn s1_iban_mod97_passing_iban_emits_detection() {
let pipeline = validator_pipeline(
r"\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b",
ValidatorKind::IbanMod97,
Some(NormalizerKind::IbanCanonical),
);
let session = Session::new(Scope::Ephemeral).expect("session");
let input = "IBAN: GB82WEST12345698765432";
let clean = clean_text(&pipeline, &session, input);
assert!(clean.starts_with("IBAN: <"), "{clean}");
assert!(clean.ends_with(":Custom:credit_card_or_iban_1>"), "{clean}");
assert_eq!(restore_tokens(&session, &clean), input);
}
#[test]
fn s1_iban_mod97_failing_iban_emits_no_detection() {
let pipeline = validator_pipeline(
r"\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b",
ValidatorKind::IbanMod97,
Some(NormalizerKind::IbanCanonical),
);
let session = Session::new(Scope::Ephemeral).expect("session");
let input = "IBAN: GB99WEST12345698765432";
let clean = clean_text(&pipeline, &session, input);
assert_eq!(clean, input);
}
#[test]
fn aadhaar_verhoeff_accepts_generated_values_and_rejects_mutants() {
assert!(ValidatorKind::AadhaarVerhoeff.validates("2345 6789 0124"));
let values = (20..30).map(gen_aadhaar).collect::<Vec<_>>();
for value in &values {
assert!(ValidatorKind::AadhaarVerhoeff.validates(value), "{value}");
assert!(gaze_recognizers::validators::verhoeff::validate_aadhaar(
value
));
assert!(!ValidatorKind::AadhaarVerhoeff.validates(&flip_last_digit(value)));
}
assert_validator_pipeline_round_trip(
r"\b\d{4} \d{4} \d{4}\b",
ValidatorKind::AadhaarVerhoeff,
"Aadhaar 6741 8529 6303",
"aadhaar",
);
}
#[test]
fn fr_nir_mod97_accepts_generated_values_and_rejects_mutants() {
for value in ["151024610204325", "1 84 12 76 451 089 46"] {
assert!(ValidatorKind::FrNirMod97.validates(value), "{value}");
}
let values = (0..10).map(gen_fr_nir).collect::<Vec<_>>();
for value in &values {
assert!(ValidatorKind::FrNirMod97.validates(value), "{value}");
assert!(gaze_recognizers::validators::mod97_nir::validate_fr_nir(
value
));
assert!(!ValidatorKind::FrNirMod97.validates(&flip_last_digit(value)));
}
assert_validator_pipeline_round_trip(
r"\b[12]\d(?: ?\d){13}\b",
ValidatorKind::FrNirMod97,
"NIR 190010100100058",
"nir",
);
}
#[test]
fn de_steuer_id_mod1110_accepts_generated_values_and_rejects_mutants() {
for value in ["48954371207", "55492670836", "65929970489"] {
assert!(ValidatorKind::DeSteuerIdMod1110.validates(value), "{value}");
}
assert!(!ValidatorKind::DeSteuerIdMod1110.validates("12345678903"));
let values = (0..10).map(gen_steuer_id).collect::<Vec<_>>();
for value in &values {
assert!(ValidatorKind::DeSteuerIdMod1110.validates(value), "{value}");
assert!(gaze_recognizers::validators::mod11::validate_de_steuer_id(
value
));
assert!(!ValidatorKind::DeSteuerIdMod1110.validates(&flip_last_digit(value)));
}
assert_validator_pipeline_round_trip(
r"\b\d{2} \d{3} \d{3} \d{3}\b",
ValidatorKind::DeSteuerIdMod1110,
"Steuer-ID 48 954 371 207",
"steuer_id",
);
}
#[test]
fn bsn_mod11_accepts_generated_values_and_rejects_mutants() {
assert!(ValidatorKind::BsnMod11.validates("123456782"));
let values = (0..10).map(gen_bsn).collect::<Vec<_>>();
for value in &values {
assert!(ValidatorKind::BsnMod11.validates(value), "{value}");
assert!(gaze_recognizers::validators::mod11::validate_bsn(value));
assert!(!ValidatorKind::BsnMod11.validates(&flip_last_digit(value)));
}
assert_validator_pipeline_round_trip(
r"\b\d{9}\b",
ValidatorKind::BsnMod11,
"BSN 111222333",
"bsn",
);
}
#[test]
fn cpf_mod11_accepts_generated_values_and_rejects_mutants() {
let values = (0..10).map(gen_cpf).collect::<Vec<_>>();
for value in &values {
assert!(ValidatorKind::CpfMod11.validates(value), "{value}");
assert!(gaze_recognizers::validators::mod11::validate_cpf(value));
assert!(!ValidatorKind::CpfMod11.validates(&flip_last_digit(value)));
}
assert_validator_pipeline_round_trip(
r"\b\d{3}\.\d{3}\.\d{3}-\d{2}\b",
ValidatorKind::CpfMod11,
"CPF 529.982.247-25",
"cpf",
);
}
#[test]
fn cnpj_mod11_accepts_generated_values_and_rejects_mutants() {
let values = (0..10).map(gen_cnpj).collect::<Vec<_>>();
for value in &values {
assert!(ValidatorKind::CnpjMod11.validates(value), "{value}");
assert!(gaze_recognizers::validators::mod11::validate_cnpj(value));
assert!(!ValidatorKind::CnpjMod11.validates(&flip_last_digit(value)));
}
assert_validator_pipeline_round_trip(
r"\b\d{2}\.\d{3}\.\d{3}/\d{4}-\d{2}\b",
ValidatorKind::CnpjMod11,
"CNPJ 04.252.011/0001-10",
"cnpj",
);
}
#[test]
fn uk_nhs_mod11_accepts_generated_values_and_rejects_mutants() {
let values = (0..10).map(gen_nhs).collect::<Vec<_>>();
for value in &values {
assert!(ValidatorKind::UkNhsMod11.validates(value), "{value}");
assert!(gaze_recognizers::validators::mod11::validate_uk_nhs(value));
assert!(!ValidatorKind::UkNhsMod11.validates(&flip_last_digit(value)));
}
assert_validator_pipeline_round_trip(
r"\b\d{3} \d{3} \d{4}\b",
ValidatorKind::UkNhsMod11,
"NHS number 943 476 5919",
"nhs_number",
);
}
fn assert_validator_pipeline_round_trip(
pattern: &str,
validator: ValidatorKind,
input: &str,
class: &str,
) {
let pipeline = custom_validator_pipeline(pattern, PiiClass::custom(class), validator, None);
let session = Session::new(Scope::Ephemeral).expect("session");
let clean = clean_text(&pipeline, &session, input);
assert!(clean.contains(&format!(":Custom:{class}_1>")), "{clean}");
assert_eq!(restore_tokens(&session, &clean), input);
}
fn flip_last_digit(value: &str) -> String {
let mut bytes = value.as_bytes().to_vec();
let index = bytes
.iter()
.rposition(u8::is_ascii_digit)
.expect("digit fixture");
bytes[index] = if bytes[index] == b'9' {
b'0'
} else {
bytes[index] + 1
};
String::from_utf8(bytes).expect("ascii fixture")
}
fn gen_aadhaar(seed: u64) -> String {
let mut digits = [0u8; 12];
digits[0] = 2 + (seed as u8 % 8);
for (index, digit) in digits[..11].iter_mut().enumerate().skip(1) {
*digit = ((seed + index as u64 * 7) % 10) as u8;
}
digits[11] = verhoeff_check_digit(&digits[..11]);
digits_to_string(&digits)
}
fn verhoeff_check_digit(digits: &[u8]) -> u8 {
const D: [[u8; 10]; 10] = [
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
[1, 2, 3, 4, 0, 6, 7, 8, 9, 5],
[2, 3, 4, 0, 1, 7, 8, 9, 5, 6],
[3, 4, 0, 1, 2, 8, 9, 5, 6, 7],
[4, 0, 1, 2, 3, 9, 5, 6, 7, 8],
[5, 9, 8, 7, 6, 0, 4, 3, 2, 1],
[6, 5, 9, 8, 7, 1, 0, 4, 3, 2],
[7, 6, 5, 9, 8, 2, 1, 0, 4, 3],
[8, 7, 6, 5, 9, 3, 2, 1, 0, 4],
[9, 8, 7, 6, 5, 4, 3, 2, 1, 0],
];
const P: [[u8; 10]; 8] = [
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
[1, 5, 7, 6, 2, 8, 3, 0, 9, 4],
[5, 8, 0, 3, 7, 9, 6, 1, 4, 2],
[8, 9, 1, 6, 0, 4, 3, 5, 2, 7],
[9, 4, 5, 3, 1, 2, 6, 8, 7, 0],
[4, 2, 8, 6, 5, 7, 3, 9, 0, 1],
[2, 7, 9, 3, 8, 0, 6, 4, 1, 5],
[7, 0, 4, 6, 9, 1, 3, 2, 5, 8],
];
const INV: [u8; 10] = [0, 4, 3, 2, 1, 5, 6, 7, 8, 9];
let mut checksum = 0u8;
for (index, digit) in digits.iter().rev().enumerate() {
checksum = D[checksum as usize][P[(index + 1) % 8][*digit as usize] as usize];
}
INV[checksum as usize]
}
fn gen_fr_nir(seed: u64) -> String {
let base = format!("1{:02}0101001{:03}", 90 + seed % 10, seed % 999);
let number = base.parse::<u64>().expect("nir base");
format!("{base}{:02}", 97 - (number % 97))
}
fn gen_steuer_id(seed: u64) -> String {
let mut digits = [0u8; 11];
for (index, digit) in digits[..10].iter_mut().enumerate() {
*digit = ((seed + index as u64) % 10) as u8;
}
digits[0] = 1 + (digits[0] % 9);
digits[10] = steuer_id_check_digit(&digits[..10]);
digits_to_string(&digits)
}
fn steuer_id_check_digit(digits: &[u8]) -> u8 {
let mut product = 10u8;
for digit in digits {
let mut sum = (*digit + product) % 10;
if sum == 0 {
sum = 10;
}
product = (2 * sum) % 11;
}
(11 - product) % 10
}
fn gen_bsn(seed: u64) -> String {
for candidate in seed * 100.. {
let mut digits = first_digits::<8>(candidate);
digits[0] = 1 + (digits[0] % 9);
let sum: u32 = digits
.iter()
.enumerate()
.map(|(index, digit)| u32::from(*digit) * (9 - index as u32))
.sum();
let check = sum % 11;
if check < 10 {
let mut full = [0u8; 9];
full[..8].copy_from_slice(&digits);
full[8] = check as u8;
return digits_to_string(&full);
}
}
unreachable!("unbounded BSN generator finds a check digit")
}
fn gen_cpf(seed: u64) -> String {
let mut digits = [0u8; 11];
digits[..9].copy_from_slice(&first_digits::<9>(seed + 529_982_247));
digits[9] = cpf_check_digit(&digits[..9], 10);
digits[10] = cpf_check_digit(&digits[..10], 11);
digits_to_string(&digits)
}
fn gen_cnpj(seed: u64) -> String {
let mut digits = [0u8; 14];
digits[..12].copy_from_slice(&first_digits::<12>(seed + 42_520_110_001));
digits[12] = weighted_check_digit(&digits[..12], &[5, 4, 3, 2, 9, 8, 7, 6, 5, 4, 3, 2]);
digits[13] = weighted_check_digit(&digits[..13], &[6, 5, 4, 3, 2, 9, 8, 7, 6, 5, 4, 3, 2]);
digits_to_string(&digits)
}
fn gen_nhs(seed: u64) -> String {
for candidate in seed * 100.. {
let digits = first_digits::<9>(candidate + 943_476_591);
let sum: u32 = digits
.iter()
.enumerate()
.map(|(index, digit)| u32::from(*digit) * (10 - index as u32))
.sum();
let check = 11 - (sum % 11);
let check = if check == 11 { 0 } else { check };
if check != 10 {
let mut full = [0u8; 10];
full[..9].copy_from_slice(&digits);
full[9] = check as u8;
return digits_to_string(&full);
}
}
unreachable!("unbounded NHS generator finds a check digit")
}
fn first_digits<const N: usize>(mut value: u64) -> [u8; N] {
let mut digits = [0u8; N];
for digit in digits.iter_mut().rev() {
*digit = (value % 10) as u8;
value /= 10;
}
digits
}
fn cpf_check_digit(digits: &[u8], start_weight: u8) -> u8 {
let sum: u32 = digits
.iter()
.zip((2..=start_weight).rev())
.map(|(digit, weight)| u32::from(*digit) * u32::from(weight))
.sum();
let remainder = sum % 11;
if remainder < 2 {
0
} else {
(11 - remainder) as u8
}
}
fn weighted_check_digit(digits: &[u8], weights: &[u8]) -> u8 {
let sum: u32 = digits
.iter()
.zip(weights)
.map(|(digit, weight)| u32::from(*digit) * u32::from(*weight))
.sum();
let remainder = sum % 11;
if remainder < 2 {
0
} else {
(11 - remainder) as u8
}
}
fn digits_to_string(digits: &[u8]) -> String {
digits
.iter()
.map(|digit| char::from(b'0' + *digit))
.collect()
}