use super::*;
pub(crate) fn header_corroborates_coordinate(header: &str) -> bool {
let h = header.to_lowercase();
if header_hint(&h).is_some_and(|t| t.starts_with("geography.coordinate")) {
return true;
}
if h.contains("latitude")
|| h.contains("longitude")
|| h.contains("coord")
|| h.contains("wgs84")
|| h.contains("northing")
|| h.contains("easting")
{
return true;
}
h.split(|c: char| !c.is_alphanumeric()).any(|tok| {
matches!(
tok,
"lat" | "lon" | "lng" | "long" | "lats" | "lons" | "latdd" | "londd" | "gps"
)
})
}
pub(crate) const STATE_CODES: &[&str] = &[
"AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA", "HI", "ID", "IL", "IN", "IA", "KS",
"KY", "LA", "ME", "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY",
"NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV",
"WI", "WY", "DC", "AB", "BC", "MB", "NB", "NL", "NS", "ON", "PE", "QC", "SK", "NT", "NU", "YT",
"NSW", "VIC", "QLD", "WA", "SA", "TAS", "ACT",
];
pub(crate) fn header_corroborates_state(header: &str) -> bool {
header
.to_lowercase()
.split(|c: char| !c.is_alphanumeric())
.any(|tok| matches!(tok, "state" | "states" | "province" | "provinces" | "prov"))
}
pub(crate) fn header_corroborates_timezone(header: &str) -> bool {
let h = header.to_lowercase();
if h.contains("timezone") || h.contains("time zone") {
return true;
}
h.split(|c: char| !c.is_alphanumeric())
.any(|tok| matches!(tok, "tz" | "tzname" | "tzabbr" | "zone"))
}
pub(crate) fn header_corroborates_isbn(header: &str) -> bool {
header.to_lowercase().contains("isbn")
}
pub(crate) fn header_corroborates_naics(header: &str) -> bool {
header
.to_lowercase()
.split(|c: char| !c.is_alphabetic())
.any(|tok| tok == "naics")
}
pub(crate) fn header_corroborates_imei(header: &str) -> bool {
header
.to_lowercase()
.split(|c: char| !c.is_alphanumeric())
.any(|tok| tok == "imei")
}
pub(crate) fn header_corroborates_cpt(header: &str) -> bool {
header
.to_lowercase()
.split(|c: char| !c.is_alphabetic())
.any(|tok| matches!(tok, "cpt" | "procedure" | "procedures"))
}
pub(crate) fn header_corroborates_hs_code(header: &str) -> bool {
header
.to_lowercase()
.split(|c: char| !c.is_alphanumeric())
.any(|tok| {
matches!(
tok,
"hs" | "hscode"
| "hscodes"
| "hts"
| "htsus"
| "htscode"
| "harmonized"
| "harmonised"
| "tariff"
| "tariffs"
| "customs"
| "commodity"
)
})
}
pub(crate) fn header_corroborates_tld(header: &str) -> bool {
header
.to_lowercase()
.split(|c: char| !c.is_alphanumeric())
.any(|tok| tok == "tld" || tok == "tlds")
}
pub(crate) fn header_corroborates_ticker(header: &str) -> bool {
header
.to_lowercase()
.split(|c: char| !c.is_alphanumeric())
.any(|tok| matches!(tok, "ticker" | "tickers" | "symbol" | "symbols"))
}
pub(crate) fn header_corroborates_numeric_code(header: &str) -> bool {
let lower = header.to_lowercase();
let mut code_tok = false;
let mut postal_tok = false;
for tok in lower.split(|c: char| !c.is_alphanumeric()) {
match tok {
"code" | "codes" | "naics" | "sic" | "cik" | "fips" => code_tok = true,
"zip" | "postal" | "postcode" | "zipcode" | "pin" | "pincode" | "plz" | "cep" => {
postal_tok = true
}
_ => {}
}
}
code_tok && !postal_tok
}
pub(crate) fn header_corroborates_region(header: &str) -> bool {
header
.to_lowercase()
.split(|c: char| !c.is_alphanumeric())
.any(|tok| {
matches!(
tok,
"region"
| "regions"
| "county"
| "counties"
| "district"
| "districts"
| "borough"
| "boroughs"
| "province"
| "provinces"
| "prefecture"
| "oblast"
| "canton"
| "governorate"
| "department"
| "division"
)
})
}
pub(crate) fn values_look_like_state_codes(sample: &[String]) -> bool {
let (mut hit, mut total) = (0usize, 0usize);
for v in sample.iter().take(16) {
let t = v.trim();
if t.is_empty() {
continue;
}
total += 1;
let up = t.to_ascii_uppercase();
if STATE_CODES.contains(&up.as_str()) {
hit += 1;
}
}
total >= 3 && hit * 100 >= total * 80
}
pub(crate) fn values_look_like_bare_numbers(sample: &[String]) -> (bool, bool) {
let (mut bare, mut total, mut any_decimal) = (0usize, 0usize, false);
for v in sample.iter().take(16) {
let t = v.trim();
if t.is_empty() {
continue;
}
total += 1;
let body = t.strip_prefix('-').unwrap_or(t);
let mut dots = 0usize;
let is_bare = !body.is_empty()
&& body.chars().all(|c| {
if c == '.' {
dots += 1;
true
} else {
c.is_ascii_digit()
}
})
&& dots <= 1;
if is_bare {
bare += 1;
if dots == 1 {
any_decimal = true;
}
}
}
(total >= 3 && bare * 100 >= total * 80, any_decimal)
}
pub(crate) fn values_are_clearly_non_url(sample: &[String]) -> bool {
let (mut urlish, mut total) = (0usize, 0usize);
for v in sample.iter().take(16) {
let t = v.trim();
if t.is_empty() {
continue;
}
total += 1;
if t.contains("://") || t.starts_with('/') {
urlish += 1;
}
}
total >= 3 && urlish * 2 < total
}
pub(crate) fn values_look_like_generic_decimals(sample: &[String]) -> bool {
let (mut numeric, mut total) = (0usize, 0usize);
for v in sample.iter().take(8) {
let t = v.trim();
if t.is_empty() {
continue;
}
total += 1;
if t.parse::<f64>().is_ok() {
numeric += 1;
}
}
total > 0 && numeric * 100 >= total * 80
}
pub(crate) fn header_corroborates_postal(header: &str) -> bool {
let h = header.to_lowercase();
if h.contains("postal") || h.contains("postcode") || h.contains("pincode") {
return true;
}
let mut has_post = false;
let mut has_code = false;
for tok in h.split(|c: char| !c.is_alphanumeric()) {
match tok {
"zip" | "zipcode" | "zip4" | "plz" | "cep" => return true,
"post" => has_post = true,
"code" => has_code = true,
_ => {}
}
}
has_post && has_code
}
pub(crate) fn values_look_like_generic_integers(sample: &[String]) -> bool {
let (mut bare_int, mut total) = (0usize, 0usize);
for v in sample.iter().take(8) {
let t = v.trim();
if t.is_empty() {
continue;
}
total += 1;
let digits = t.strip_prefix('-').unwrap_or(t);
if !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit()) {
if digits.len() > 1 && digits.starts_with('0') {
return false; }
bare_int += 1;
}
}
total > 0 && bare_int * 100 >= total * 80
}
pub(crate) fn header_hint(header: &str) -> Option<&'static str> {
let h = header.to_lowercase();
let h = h.replace(['_', '-'], " ");
let h = h.trim();
let disable_table = rhh::is_disabled("header_hint_table");
let disable_identity = rhh::is_disabled("substring_matcher_identity");
let disable_technology = rhh::is_disabled("substring_matcher_technology");
let disable_geography = rhh::is_disabled("substring_matcher_geography");
let disable_datetime = rhh::is_disabled("substring_matcher_datetime");
let disable_finance = rhh::is_disabled("substring_matcher_finance");
let disable_representation = rhh::is_disabled("substring_matcher_representation");
if !disable_table {
match h {
"email" | "e mail" | "email address" | "emailaddress" => {
return Some("identity.person.email");
}
"url" | "uri" | "link" | "href" | "website" | "homepage" => {
return Some("technology.internet.url");
}
"ip" | "ip address" | "ipaddress" | "ip addr" | "source ip" | "destination ip"
| "src ip" | "dst ip" | "server ip" | "client ip" | "remote ip" | "local ip" => {
return Some("technology.internet.ip_v4");
}
"uuid" | "guid" => {
return Some("technology.identifier.uuid");
}
"gender" | "sex" => {
return Some("identity.person.gender");
}
"age" | "patient age" | "customer age" | "user age" => {
return Some("representation.numeric.integer_number");
}
"epoch" | "unix epoch" | "unix timestamp" | "epoch time" | "unix time"
| "epoch seconds" | "unix seconds" | "epoch s" | "posix time"
| "unix epoch seconds" => {
return Some("datetime.epoch.unix_seconds");
}
"epoch ms" | "unix ms" | "epoch milliseconds" | "unix milliseconds" => {
return Some("datetime.epoch.unix_milliseconds");
}
"altitude" | "elevation" | "alt" | "elev" => {
return Some("representation.numeric.integer_number");
}
"duration minutes" | "duration min" | "duration seconds" | "duration sec"
| "duration hours" | "duration hrs" | "duration ms" | "elapsed" | "elapsed time" => {
return Some("representation.numeric.integer_number");
}
"attendance" | "headcount" | "participants" | "crowd size" | "capacity" => {
return Some("representation.numeric.integer_number");
}
"heart rate" | "heartrate" | "hr" | "bpm" | "pulse" => {
return Some("representation.numeric.integer_number");
}
"pages" | "page count" | "num pages" | "total pages" => {
return Some("representation.numeric.integer_number");
}
"language" | "lang" | "programming language" | "spoken language" => {
return Some("representation.text.word");
}
"sport" | "discipline" | "event type" | "game type" => {
return Some("representation.text.word");
}
"species" | "genus" | "taxon" | "breed" | "variety" => {
return Some("representation.text.word");
}
"exchange" | "stock exchange" | "market" | "bourse" => {
return Some("representation.text.word");
}
"latitude" | "lat" => {
return Some("geography.coordinate.latitude");
}
"longitude" | "lng" | "lon" | "long" => {
return Some("geography.coordinate.longitude");
}
"country" | "country name" => {
return Some("geography.location.country");
}
"country code" | "alpha 2" | "alpha 3" | "iso country" | "iso alpha 2"
| "iso alpha 3" | "country iso" => {
return Some("geography.location.country_code");
}
"city" | "city name" => {
return Some("geography.location.city");
}
"state" | "province" => {
return Some("geography.location.region");
}
"currency" | "currency code" => {
return Some("identity.financial.currency_code");
}
"sibsp" | "parch" | "siblings" | "parents" | "children" | "dependents" | "qty"
| "quantity" => {
return Some("representation.numeric.integer_number");
}
"survived" | "alive" | "deceased" | "dead" | "active" | "enabled" | "disabled"
| "deleted" | "verified" | "approved" | "flagged" => {
return Some("representation.boolean.binary");
}
"utc offset" | "gmt offset" | "timezone offset" | "tz offset" | "utcoffset"
| "gmtoffset" => {
return Some("datetime.offset.utc");
}
"timezone" | "tz" | "time zone" | "iana timezone" | "iana tz" | "olson timezone" => {
return Some("datetime.offset.iana");
}
"publisher" | "publishing house" | "published by" | "company" | "employer"
| "organization" | "organisation" | "venue" | "stadium" | "arena" | "theater"
| "theatre" | "station" | "station name" | "facility" | "building" | "hotel"
| "restaurant" | "hospital" | "school" | "university" | "manufacturer" => {
return Some("representation.text.entity_name");
}
"swift" | "swift code" | "bic" | "bic code" | "swiftcode" | "biccode" => {
return Some("finance.banking.swift_bic");
}
"issn" => {
return Some("identity.commerce.issn");
}
"icao" | "icao code" => {
return Some("geography.transportation.icao_code");
}
"author" | "authors" | "author name" => {
return Some("identity.person.full_name");
}
"npi" | "npi number" => {
return Some("identity.medical.npi");
}
"ean" | "barcode" | "gtin" => {
return Some("identity.commerce.ean");
}
"upc" => {
return Some("identity.commerce.upc");
}
"os" | "operating system" | "platform" => {
return Some("technology.development.os");
}
"occupation" | "job title" | "jobtitle" | "job" | "profession" | "role"
| "position" => {
return Some("representation.text.word");
}
"embarked" | "boarded" | "departed" | "terminal" | "gate" => {
return Some("representation.text.word");
}
"ticket" | "ticket number" | "ticketno" => {
return Some("representation.identifier.alphanumeric_id");
}
"cabin" | "room" | "compartment" | "berth" | "seat" => {
return Some("representation.identifier.alphanumeric_id");
}
"fare" | "fee" | "toll" | "charge" => {
return Some("finance.currency.amount");
}
"amount accounting" | "amount acc" => {
return Some("finance.currency.amount_accounting");
}
"amount apostrophe" => {
return Some("finance.currency.amount_apostrophe");
}
"amount code prefix" | "amount code" | "amount with code" => {
return Some("finance.currency.amount_code_prefix");
}
"amount comma" => {
return Some("finance.currency.amount_comma");
}
"amount comma suffix" => {
return Some("finance.currency.amount_comma_suffix");
}
"amount crypto" | "crypto amount" => {
return Some("finance.currency.amount_crypto");
}
"amount lakh" | "lakh amount" | "amount indian" => {
return Some("finance.currency.amount_lakh");
}
"amount multisym" | "amount multi sym" | "amount multiple symbols" => {
return Some("finance.currency.amount_multisym");
}
"amount neg trailing" | "amount negative trailing" => {
return Some("finance.currency.amount_neg_trailing");
}
"amount nodecimal" | "amount no decimal" => {
return Some("finance.currency.amount_nodecimal");
}
"amount space" => {
return Some("finance.currency.amount_space");
}
_ => {}
}
}
if !disable_identity && (h.contains("email") || h.contains("e mail")) && !h.contains("display")
{
return Some("identity.person.email");
}
if !disable_identity
&& (h.contains("phone") || h.contains("tel") || h.contains("mobile") || h.contains("fax"))
&& !h.contains("e164")
{
return Some("identity.person.phone_number");
}
if !disable_technology && (h.contains("v6") || h.contains("ipv6")) {
return Some("technology.internet.ip_v6");
}
if !disable_technology
&& (h.ends_with(" ip") || h.starts_with("ip ") || h.contains(" ip "))
&& !h.contains("port")
{
return Some("technology.internet.ip_v4");
}
if !disable_geography && (h.contains("zip") || h.contains("postal") || h.contains("postcode")) {
return Some("geography.address.postal_code");
}
if !disable_identity && h.contains("name") && (h.contains("first") || h.contains("given")) {
return Some("identity.person.first_name");
}
if !disable_identity
&& h.contains("name")
&& (h.contains("last") || h.contains("family") || h.contains("sur"))
{
return Some("identity.person.last_name");
}
if !disable_identity && h.contains("name") && (h.contains("full") || h.contains("complete")) {
return Some("identity.person.full_name");
}
if !disable_geography
&& h.contains("address")
&& !h.contains("email")
&& !h.contains("ip")
&& !h.contains("mac")
&& !h.contains("bitcoin")
&& !h.contains("btc")
&& !h.contains("crypto")
&& !h.contains("wallet")
{
if h.contains("street") {
return Some("geography.address.street_address");
}
return Some("geography.address.full_address");
}
if !disable_geography && h.contains("street") {
if h.contains("suffix") {
return Some("geography.address.street_suffix");
}
if h.contains("name") {
return Some("geography.address.street_name");
}
return Some("geography.address.street_address");
}
if !disable_datetime && (h.contains("born") || h.contains("birth") || h.contains("dob")) {
return Some("datetime.date.iso");
}
if !disable_datetime && (h.contains("rfc 2822") || h.contains("rfc2822")) {
return Some("datetime.timestamp.rfc_2822");
}
if !disable_datetime && (h.contains("rfc 3339") || h.contains("rfc3339")) {
return Some("datetime.timestamp.rfc_3339");
}
if !disable_datetime && h.contains("sql") && (h.contains("timestamp") || h.contains("datetime"))
{
return Some("datetime.timestamp.sql_standard");
}
if !disable_datetime && (h.contains("epoch") || h.contains("unix")) {
if h.contains("ms") || h.contains("milli") {
return Some("datetime.epoch.unix_milliseconds");
}
return Some("datetime.epoch.unix_seconds");
}
if !disable_datetime
&& (h.contains("date") || h.contains("timestamp") || h.contains("datetime"))
&& !h.contains("month")
{
return Some("datetime.timestamp.iso_8601");
}
if !disable_datetime && h.contains("year") {
return Some("datetime.component.year");
}
if !disable_identity && h.contains("weight") {
return Some("identity.person.weight");
}
if !disable_identity && h.contains("height") {
return Some("identity.person.height");
}
if !disable_identity && (h.contains("password") || h.contains("passwd")) {
return Some("identity.person.password");
}
if !disable_technology && (h.contains("url") || h.contains("link") || h.contains("href")) {
return Some("technology.internet.url");
}
if !disable_finance
&& (h.contains("price")
|| h.contains("cost")
|| h.contains("amount")
|| h.contains("salary")
|| h.contains("revenue")
|| h.contains("income")
|| h.contains("wage")
|| h.contains("budget")
|| h.contains("expense"))
{
return Some("finance.currency.amount");
}
if !disable_representation
&& ((h.contains("count") && !h.contains("country") && !h.contains("county"))
|| h.contains("quantity")
|| h.contains("num ")
|| h.starts_with("num")
|| h.ends_with(" num"))
{
return Some("representation.numeric.integer_number");
}
if !disable_representation
&& (h.contains("ticket") || h.contains("cabin") || h.contains("seat") || h.contains("room"))
{
return Some("representation.identifier.alphanumeric_id");
}
if !disable_finance
&& (h.contains("fare") || h.contains("fee") || h.contains("charge") || h.contains("toll"))
{
return Some("finance.currency.amount");
}
if !disable_representation
&& (h.contains("pressure")
|| h.contains("temperature")
|| h.contains("voltage")
|| h.contains("density")
|| h.contains("velocity")
|| h.contains("acceleration")
|| h.contains("frequency")
|| h.contains("resistance")
|| h.contains("capacitance")
|| h.contains("inductance"))
{
return Some("representation.numeric.decimal_number");
}
if !disable_representation
&& (h.contains("latency")
|| h.contains("elapsed")
|| (h.contains("duration") && !h.contains("iso") && !h.contains("8601")))
{
return Some("representation.numeric.integer_number");
}
if !disable_representation
&& (h.contains("payload") || h.contains("bytes") || h.contains("size"))
{
return Some("representation.numeric.integer_number");
}
None
}