use super::super::*;
#[test]
fn test_numeric_sequential_detection() {
let values: Vec<String> = vec!["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]
.into_iter()
.map(String::from)
.collect();
let results: Vec<ClassificationResult> = values
.iter()
.map(|_| ClassificationResult {
label: "representation.identifier.increment".to_string(),
confidence: 0.8,
all_scores: vec![],
})
.collect();
let votes = [
("representation.identifier.increment".to_string(), 8),
("representation.numeric.integer_number".to_string(), 2),
];
let top_labels: Vec<&str> = votes.iter().map(|(l, _)| l.as_str()).collect();
let result = disambiguate_numeric(&values, &results, &top_labels);
assert!(result.is_some());
let (label, _rule) = result.unwrap();
assert_eq!(label, "representation.identifier.increment");
}
#[test]
fn disambiguate_numeric_no_overflow_on_min_sentinel() {
let values: Vec<String> = vec![
i64::MIN.to_string(),
"1".to_string(),
"2".to_string(),
"4".to_string(),
"8".to_string(),
"16".to_string(),
];
let results: Vec<ClassificationResult> = values
.iter()
.map(|_| ClassificationResult {
label: "representation.numeric.integer_number".to_string(),
confidence: 0.8,
all_scores: vec![],
})
.collect();
let top_labels = ["representation.numeric.integer_number"];
let result = disambiguate_numeric(&values, &results, &top_labels);
assert!(result.is_none());
}
#[test]
fn increment_substance_veto_is_default_on() {
assert!(!rhh::is_disabled("increment_substance_veto"));
}
#[test]
fn values_form_increment_accepts_genuine_auto_increment() {
let vals: Vec<String> = (1..=200).map(|i| i.to_string()).collect();
assert_eq!(values_form_increment(&vals), Some(true));
let gapped: Vec<String> = (1..=200)
.filter(|i| i % 10 != 0)
.map(|i| i.to_string())
.collect();
assert_eq!(values_form_increment(&gapped), Some(true));
}
#[test]
fn values_form_increment_rejects_evenly_spaced_non_increment() {
let spaced: Vec<String> = (1..=50).map(|i| (i * 100).to_string()).collect();
assert_eq!(values_form_increment(&spaced), Some(false));
let measures: Vec<String> = ["3", "7", "3", "42", "7", "108", "3", "9", "256", "42"]
.into_iter()
.map(String::from)
.collect();
assert_eq!(values_form_increment(&measures), Some(false));
}
#[test]
fn values_form_increment_abstains_on_too_few_or_nonnumeric() {
assert_eq!(
values_form_increment(&["1".into(), "2".into(), "3".into()]),
None
);
let words: Vec<String> = ["a", "b", "c", "d", "e", "f"]
.into_iter()
.map(String::from)
.collect();
assert_eq!(values_form_increment(&words), None);
}
#[test]
fn test_si_number_override_plain_decimals() {
let values: Vec<String> = vec!["5.1", "3.5", "1.4", "7.9", "0.2", "4.6"]
.into_iter()
.map(String::from)
.collect();
let result = disambiguate_si_number(&values);
assert_eq!(
result,
Some((
"representation.numeric.decimal_number".to_string(),
"si_number_override_no_suffix".to_string()
))
);
}
#[test]
fn test_si_number_override_real_si_values() {
let values: Vec<String> = vec!["5.1K", "3.5M", "1.4B"]
.into_iter()
.map(String::from)
.collect();
let result = disambiguate_si_number(&values);
assert_eq!(result, None);
}
#[test]
fn test_si_number_override_mixed_values() {
let values: Vec<String> = vec!["5.1", "3.5K", "1.4"]
.into_iter()
.map(String::from)
.collect();
let result = disambiguate_si_number(&values);
assert_eq!(result, None);
}
#[test]
fn test_si_number_override_negative_decimals() {
let values: Vec<String> = vec!["-450.12", "732.57", "-1.003", "98.6"]
.into_iter()
.map(String::from)
.collect();
let result = disambiguate_si_number(&values);
assert_eq!(
result,
Some((
"representation.numeric.decimal_number".to_string(),
"si_number_override_no_suffix".to_string()
))
);
}
#[test]
fn test_si_number_override_empty_values() {
let values: Vec<String> = vec!["", " ", ""].into_iter().map(String::from).collect();
let result = disambiguate_si_number(&values);
assert_eq!(
result,
Some((
"representation.numeric.decimal_number".to_string(),
"si_number_override_no_suffix".to_string()
))
);
}
#[test]
fn test_rule19_percentage_no_sign_demotes_to_decimal() {
let values: Vec<String> = vec!["5.1", "3.5", "1.4", "0.2", "4.9"]
.into_iter()
.map(String::from)
.collect();
let results: Vec<ClassificationResult> = values
.iter()
.map(|_| ClassificationResult {
label: "representation.numeric.percentage".to_string(),
confidence: 0.8,
all_scores: vec![],
})
.collect();
let votes = vec![
("representation.numeric.percentage".to_string(), 4),
("representation.numeric.decimal_number".to_string(), 1),
];
let result = disambiguate(&values, &results, &votes, 5, None);
assert!(
result.is_some(),
"Rule 19 should fire for percentage without '%'"
);
let (label, rule) = result.unwrap();
assert_eq!(label, "representation.numeric.decimal_number");
assert_eq!(rule, "percentage_no_sign");
}
#[test]
fn test_rule19_percentage_with_sign_keeps_percentage() {
let values: Vec<String> = vec!["35.36%", "12.5%", "98.1%", "0.5%", "100%"]
.into_iter()
.map(String::from)
.collect();
let results: Vec<ClassificationResult> = values
.iter()
.map(|_| ClassificationResult {
label: "representation.numeric.percentage".to_string(),
confidence: 0.9,
all_scores: vec![],
})
.collect();
let votes = vec![("representation.numeric.percentage".to_string(), 5)];
let result = disambiguate(&values, &results, &votes, 5, None);
assert!(
result.is_none()
|| result
.as_ref()
.map(|(_, r)| r != "percentage_no_sign")
.unwrap_or(true),
"Rule 19 should NOT fire when values contain '%'"
);
}
#[test]
fn test_aggregate_features_empty() {
let cf = aggregate_features(&[]);
assert_eq!(cf.mean, [0.0f32; FEATURE_DIM]);
assert_eq!(cf.variance, [0.0f32; FEATURE_DIM]);
}
#[test]
fn test_aggregate_features_single_value() {
let features = extract_features("abc123");
let cf = aggregate_features(&[features]);
assert_eq!(cf.mean, features);
for v in cf.variance {
assert!(
v.abs() < 1e-6,
"variance should be ~0 for single value, got {}",
v
);
}
assert_eq!(cf.min, features);
assert_eq!(cf.max, features);
}
#[test]
fn test_aggregate_features_variance() {
let f1 = extract_features("abc"); let f2 = extract_features("abcdef"); let cf = aggregate_features(&[f1, f2]);
let length_idx = feature_idx::LENGTH;
assert!((cf.mean[length_idx] - 4.5).abs() < 0.01, "mean length");
assert!(
(cf.variance[length_idx] - 2.25).abs() < 0.01,
"variance length"
);
assert!((cf.min[length_idx] - 3.0).abs() < 0.01, "min length");
assert!((cf.max[length_idx] - 6.0).abs() < 0.01, "max length");
}
#[test]
fn test_uniform_length_hex_zero_variance() {
let shas = [
"a".repeat(40),
"b1c2d3e4f5a6b7c8d9e0a1b2c3d4e5f6a7b8c9d0".to_string(),
"1234567890abcdef1234567890abcdef12345678".to_string(),
"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(),
];
let per_value: Vec<[f32; FEATURE_DIM]> = shas.iter().map(|s| extract_features(s)).collect();
let cf = aggregate_features(&per_value);
assert!(
cf.variance[feature_idx::LENGTH] < 0.01,
"uniform hex length variance should be ~0, got {}",
cf.variance[feature_idx::LENGTH]
);
assert!(
cf.mean[feature_idx::IS_HEX_STRING] >= 0.95,
"uniform hex strings should all be hex, got {}",
cf.mean[feature_idx::IS_HEX_STRING]
);
}
#[test]
fn test_hash_mixed_length_nonzero_variance() {
let hashes = [
"d41d8cd98f00b204e9800998ecf8427e".to_string(), "da39a3ee5e6b4b0d3255bfef95601890afd80709".to_string(), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".to_string(), ];
let per_value: Vec<[f32; FEATURE_DIM]> = hashes.iter().map(|s| extract_features(s)).collect();
let cf = aggregate_features(&per_value);
assert!(
cf.variance[feature_idx::LENGTH] > 100.0,
"mixed hash length variance should be high, got {}",
cf.variance[feature_idx::LENGTH]
);
assert!(
cf.mean[feature_idx::IS_HEX_STRING] >= 0.95,
"all hashes should be hex, got {}",
cf.mean[feature_idx::IS_HEX_STRING]
);
}
#[test]
fn test_rule_f5_numeric_code_demoted_without_leading_zeros() {
let mut result = ColumnResult {
label: "representation.identifier.numeric_code".to_string(),
confidence: 1.0,
vote_distribution: vec![("representation.identifier.numeric_code".to_string(), 1.0)],
disambiguation_applied: false,
disambiguation_rule: None,
samples_used: 100,
detected_locale: None,
is_generic: false,
column_features: None,
};
let mut cf = ColumnFeatures::empty();
cf.mean[feature_idx::HAS_LEADING_ZERO] = 0.0;
let votes = vec![("representation.identifier.numeric_code".to_string(), 100)];
feature_disambiguate(&mut result, &cf, &votes, 100);
assert_eq!(result.label, "representation.numeric.integer_number");
assert!(result.disambiguation_applied);
assert!(result
.disambiguation_rule
.as_ref()
.unwrap()
.starts_with("feature_no_leading_zero"));
}
#[test]
fn test_rule_f5_numeric_code_kept_with_leading_zeros() {
let mut result = ColumnResult {
label: "representation.identifier.numeric_code".to_string(),
confidence: 1.0,
vote_distribution: vec![("representation.identifier.numeric_code".to_string(), 1.0)],
disambiguation_applied: false,
disambiguation_rule: None,
samples_used: 100,
detected_locale: None,
is_generic: false,
column_features: None,
};
let mut cf = ColumnFeatures::empty();
cf.mean[feature_idx::HAS_LEADING_ZERO] = 0.5;
let votes = vec![("representation.identifier.numeric_code".to_string(), 100)];
feature_disambiguate(&mut result, &cf, &votes, 100);
assert_eq!(
result.label, "representation.identifier.numeric_code",
"should NOT demote when leading zeros are present"
);
assert!(!result.disambiguation_applied);
}
#[test]
fn test_rule_f5_numeric_code_with_decimals_becomes_decimal_number() {
let mut result = ColumnResult {
label: "representation.identifier.numeric_code".to_string(),
confidence: 1.0,
vote_distribution: vec![("representation.identifier.numeric_code".to_string(), 1.0)],
disambiguation_applied: false,
disambiguation_rule: None,
samples_used: 100,
detected_locale: None,
is_generic: false,
column_features: None,
};
let mut cf = ColumnFeatures::empty();
cf.mean[feature_idx::HAS_LEADING_ZERO] = 0.0; cf.mean[feature_idx::IS_FLOAT] = 1.0;
let votes = vec![("representation.identifier.numeric_code".to_string(), 100)];
feature_disambiguate(&mut result, &cf, &votes, 100);
assert_eq!(
result.label, "representation.numeric.decimal_number",
"numeric_code with decimal points should become decimal_number"
);
assert!(result.disambiguation_applied);
assert!(result
.disambiguation_rule
.as_ref()
.unwrap()
.starts_with("feature_decimal_over_numeric_code"));
}
#[test]
fn test_rule_f5b_decimal_demoted_to_integer_when_whole() {
let mut result = ColumnResult {
label: "representation.numeric.decimal_number".to_string(),
confidence: 1.0,
vote_distribution: vec![("representation.numeric.decimal_number".to_string(), 1.0)],
disambiguation_applied: false,
disambiguation_rule: None,
samples_used: 100,
detected_locale: None,
is_generic: false,
column_features: None,
};
let mut cf = ColumnFeatures::empty();
cf.mean[feature_idx::IS_FLOAT] = 0.0; feature_sharpen(&mut result, &cf);
assert_eq!(result.label, "representation.numeric.integer_number");
assert!(result.disambiguation_applied);
assert!(result
.disambiguation_rule
.as_ref()
.unwrap()
.starts_with("feature_decimal_to_integer_is_float"));
}
#[test]
fn test_rule_f5b_keeps_decimal_when_fractional() {
let mut result = ColumnResult {
label: "representation.numeric.decimal_number".to_string(),
confidence: 1.0,
vote_distribution: vec![("representation.numeric.decimal_number".to_string(), 1.0)],
disambiguation_applied: false,
disambiguation_rule: None,
samples_used: 100,
detected_locale: None,
is_generic: false,
column_features: None,
};
let mut cf = ColumnFeatures::empty();
cf.mean[feature_idx::IS_FLOAT] = 1.0;
feature_sharpen(&mut result, &cf);
assert_eq!(result.label, "representation.numeric.decimal_number");
assert!(!result.disambiguation_applied);
}
#[test]
fn test_rule_f3_enhanced_float_parseability() {
let mut result = ColumnResult {
label: "representation.numeric.decimal_number".to_string(),
confidence: 0.80,
vote_distribution: vec![
("representation.numeric.decimal_number".to_string(), 0.70),
("geography.transportation.hs_code".to_string(), 0.20),
],
disambiguation_applied: false,
disambiguation_rule: None,
samples_used: 100,
detected_locale: None,
is_generic: false,
column_features: None,
};
let mut cf = ColumnFeatures::empty();
cf.mean[feature_idx::DIGIT_RATIO] = 0.85;
cf.mean[feature_idx::SEGMENT_COUNT_DOT] = 1.8; cf.mean[feature_idx::IS_FLOAT] = 0.5;
let votes = vec![
("representation.numeric.decimal_number".to_string(), 70),
("geography.transportation.hs_code".to_string(), 20),
];
feature_disambiguate(&mut result, &cf, &votes, 100);
assert_eq!(result.label, "geography.transportation.hs_code");
assert!(result.disambiguation_applied);
assert!(result
.disambiguation_rule
.as_ref()
.unwrap()
.contains("feature_hs_code"));
}
#[test]
fn test_rule_f6_short_code_demoted_from_file_extension() {
let mut result = ColumnResult {
label: "representation.file.extension".to_string(),
confidence: 0.85,
vote_distribution: vec![
("representation.file.extension".to_string(), 0.70),
("representation.categorical.categorical".to_string(), 0.20),
],
disambiguation_applied: false,
disambiguation_rule: None,
samples_used: 100,
detected_locale: None,
is_generic: false,
column_features: None,
};
let mut cf = ColumnFeatures::empty();
cf.mean[feature_idx::LENGTH] = 2.5; cf.mean[feature_idx::SEGMENT_COUNT_DOT] = 1.0; cf.mean[feature_idx::ALPHA_RATIO] = 1.0;
let votes = vec![
("representation.file.extension".to_string(), 70),
("representation.categorical.categorical".to_string(), 20),
];
feature_disambiguate(&mut result, &cf, &votes, 100);
assert_eq!(result.label, "representation.categorical.categorical");
assert!(result.disambiguation_applied);
assert!(result
.disambiguation_rule
.as_ref()
.unwrap()
.contains("feature_short_code_not_extension"));
}
#[test]
fn test_rule_f6_real_extension_not_demoted() {
let mut result = ColumnResult {
label: "representation.file.extension".to_string(),
confidence: 0.90,
vote_distribution: vec![("representation.file.extension".to_string(), 1.0)],
disambiguation_applied: false,
disambiguation_rule: None,
samples_used: 100,
detected_locale: None,
is_generic: false,
column_features: None,
};
let mut cf = ColumnFeatures::empty();
cf.mean[feature_idx::LENGTH] = 4.0; cf.mean[feature_idx::SEGMENT_COUNT_DOT] = 2.0; cf.mean[feature_idx::ALPHA_RATIO] = 0.75;
let votes = vec![("representation.file.extension".to_string(), 90)];
feature_disambiguate(&mut result, &cf, &votes, 100);
assert_eq!(
result.label, "representation.file.extension",
"should NOT demote when values contain dots"
);
assert!(!result.disambiguation_applied);
}
#[test]
fn test_rule_f6_long_values_not_demoted() {
let mut result = ColumnResult {
label: "representation.file.extension".to_string(),
confidence: 0.85,
vote_distribution: vec![("representation.file.extension".to_string(), 1.0)],
disambiguation_applied: false,
disambiguation_rule: None,
samples_used: 100,
detected_locale: None,
is_generic: false,
column_features: None,
};
let mut cf = ColumnFeatures::empty();
cf.mean[feature_idx::LENGTH] = 8.0; cf.mean[feature_idx::SEGMENT_COUNT_DOT] = 1.0; cf.mean[feature_idx::ALPHA_RATIO] = 1.0;
let votes = vec![("representation.file.extension".to_string(), 85)];
feature_disambiguate(&mut result, &cf, &votes, 100);
assert_eq!(
result.label, "representation.file.extension",
"should NOT demote when mean length > 4"
);
assert!(!result.disambiguation_applied);
}
#[test]
fn test_rule_f6_fallback_to_categorical() {
let mut result = ColumnResult {
label: "representation.file.extension".to_string(),
confidence: 0.90,
vote_distribution: vec![("representation.file.extension".to_string(), 1.0)],
disambiguation_applied: false,
disambiguation_rule: None,
samples_used: 100,
detected_locale: None,
is_generic: false,
column_features: None,
};
let mut cf = ColumnFeatures::empty();
cf.mean[feature_idx::LENGTH] = 2.0;
cf.mean[feature_idx::SEGMENT_COUNT_DOT] = 1.0;
cf.mean[feature_idx::ALPHA_RATIO] = 1.0;
let votes = vec![("representation.file.extension".to_string(), 100)];
feature_disambiguate(&mut result, &cf, &votes, 100);
assert_eq!(
result.label, "representation.categorical.categorical",
"should fallback to categorical when no second vote exists"
);
assert!(result.disambiguation_applied);
}
#[test]
fn test_sharpen_f3_removed_decimal_stays_decimal() {
let mut result = ColumnResult {
label: "representation.numeric.decimal_number".to_string(),
confidence: 0.80,
vote_distribution: vec![("representation.numeric.decimal_number".to_string(), 0.80)],
disambiguation_applied: false,
disambiguation_rule: None,
samples_used: 50,
detected_locale: None,
is_generic: false,
column_features: None,
};
let mut cf = ColumnFeatures::empty();
cf.mean[feature_idx::DIGIT_RATIO] = 0.85; cf.mean[feature_idx::SEGMENT_COUNT_DOT] = 2.5; cf.mean[feature_idx::IS_FLOAT] = 0.3; cf.mean[feature_idx::HAS_NEGATIVE_PREFIX] = 0.0; cf.variance[feature_idx::SEGMENT_COUNT_DOT] = 0.1;
feature_sharpen(&mut result, &cf);
assert_eq!(
result.label, "representation.numeric.decimal_number",
"F3 removed: decimal_number should NOT be overridden to hs_code by feature_sharpen"
);
assert!(
!result.disambiguation_applied,
"No disambiguation should fire for decimal_number with HS code features"
);
}
#[test]
fn test_sharpen_f5_numeric_code_demoted_single_vote() {
let mut result = ColumnResult {
label: "representation.identifier.numeric_code".to_string(),
confidence: 0.90,
vote_distribution: vec![("representation.identifier.numeric_code".to_string(), 0.90)],
disambiguation_applied: false,
disambiguation_rule: None,
samples_used: 50,
detected_locale: None,
is_generic: false,
column_features: None,
};
let mut cf = ColumnFeatures::empty();
cf.mean[feature_idx::HAS_LEADING_ZERO] = 0.0; cf.mean[feature_idx::IS_FLOAT] = 0.0;
feature_sharpen(&mut result, &cf);
assert_eq!(
result.label, "representation.numeric.integer_number",
"F5 should demote numeric_code without leading zeros via feature_sharpen"
);
assert!(result.disambiguation_applied);
}
#[test]
fn test_sharpen_r12_numeric_single_vote() {
let values: Vec<String> = vec!["2020", "2021", "2022", "2023", "2024"]
.into_iter()
.map(String::from)
.collect();
let result = value_sharpen(&values, "representation.numeric.integer_number", 0.80, None);
assert!(result.is_some(), "R12 should fire for year-like values");
let (label, _rule) = result.unwrap();
assert_eq!(label, "datetime.component.year");
}