dataprof_core/
null_tokens.rs

1//! The text values every input path counts as null.
2//!
3//! A cell is null when it is empty or whitespace, `null` or `nan` in any case,
4//! or one of [`NULL_MARKERS`] exactly: `NA`, `N/A`, `n/a`, `#N/A`, `\N` and
5//! `None`. Typed nulls (an Arrow null, a SQL `NULL`, Python `None`, a float
6//! NaN) are null before any text is involved.
7//!
8//! Through 0.12 only empty, `null` and `nan` were nulls, so the markers the
9//! tools that write files use for a missing value were counted as values, and
10//! completeness read 100% on columns that were mostly missing (#813): `NA` is
11//! R's, `#N/A` Excel's, `\N` MySQL `SELECT INTO OUTFILE` and PostgreSQL
12//! `COPY` text format's, `None` Python's `str(None)`, `N/A` and `n/a` those of
13//! hand-maintained sheets.
14//!
15//! The markers match exactly, unlike `null` and `nan`. Each is a missing-value
16//! marker only in the spelling its writer uses: `Na` is sodium and `none` an
17//! answer, and a case-insensitive `na` would take both. The set leaves out
18//! tokens that are data in one file and a marker in the next, such as `-`,
19//! `?` and `n.d.`.
20
21/// The null vocabulary a report was measured with, as recorded in its metric
22/// semantics.
23///
24/// Only the current vocabulary has a value. Releases that knew only empty,
25/// `null` and `nan` recorded none, and a report from one reads back with the
26/// vocabulary unknown rather than as one it never declared.
27#[derive(
28    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
29)]
30#[serde(rename_all = "snake_case")]
31pub enum NullTokenSet {
32    /// Empty, `null` and `nan` in any case, and [`NULL_MARKERS`] exactly, as
33    /// matched by [`is_null_like_token`].
34    CommonMarkers,
35}
36
37/// Missing-value markers counted as null in exactly this spelling.
38pub const NULL_MARKERS: &[&str] = &["NA", "N/A", "n/a", "#N/A", "\\N", "None"];
39
40/// Whether a text value is null under [`NullTokenSet::CommonMarkers`].
41///
42/// This is the single definition behind every `null_count`, so the CSV, JSON,
43/// Parquet, Arrow, database and in-memory paths cannot drift apart on what
44/// they count as missing. Surrounding whitespace is ignored.
45pub fn is_null_like_token(value: &str) -> bool {
46    let trimmed = value.trim();
47    trimmed.is_empty()
48        || trimmed.eq_ignore_ascii_case("null")
49        || trimmed.eq_ignore_ascii_case("nan")
50        || NULL_MARKERS.contains(&trimmed)
51}
52
53#[cfg(test)]
54mod tests {
55    use super::*;
56
57    #[test]
58    fn every_marker_is_null_with_surrounding_whitespace() {
59        for marker in NULL_MARKERS {
60            assert!(is_null_like_token(marker), "{marker:?}");
61            assert!(is_null_like_token(&format!(" {marker}\t")), "{marker:?}");
62        }
63    }
64
65    #[test]
66    fn null_and_nan_match_in_any_case() {
67        for token in ["", "  ", "null", "NULL", "Null", "nan", "NaN", "NAN"] {
68            assert!(is_null_like_token(token), "{token:?}");
69        }
70    }
71
72    #[test]
73    fn markers_match_only_in_their_writers_spelling() {
74        // Sodium, an answer, and the lower-case spellings the markers' writers
75        // never produce stay values.
76        for value in [
77            "Na", "na", "none", "NONE", "#n/a", "\\n", "N/a", "NA1", "N/A/",
78        ] {
79            assert!(!is_null_like_token(value), "{value:?}");
80        }
81    }
82
83    #[test]
84    fn ambiguous_placeholders_stay_values() {
85        for value in ["-", "?", "n.d.", "nil", "(null)", "0"] {
86            assert!(!is_null_like_token(value), "{value:?}");
87        }
88    }
89}
dataprof_core/null_tokens.rs

dataprof_core/
null_tokens.rs
