dataprof_core/
semantic.rs

1use serde::{Deserialize, Serialize};
2
3use crate::errors::DataProfilerError;
4
5/// The kind of a semantic hint, used for diagnostics and binding evidence.
6#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
7#[serde(rename_all = "snake_case")]
8pub enum SemanticHintKind {
9    /// A [`SemanticHints::positive_columns`] entry.
10    Positive,
11    /// An [`SemanticHints::identifier_columns`] entry.
12    Identifier,
13    /// A [`SemanticHints::temporal_columns`] entry.
14    Temporal,
15    /// A column named by a [`SemanticHints::temporal_pairs`] entry.
16    TemporalPair,
17}
18
19impl SemanticHintKind {
20    /// The public configuration field name that carries this hint kind.
21    pub fn field_name(self) -> &'static str {
22        match self {
23            Self::Positive => "positive_columns",
24            Self::Identifier => "identifier_columns",
25            Self::Temporal => "temporal_columns",
26            Self::TemporalPair => "temporal_pairs",
27        }
28    }
29}
30
31/// Per-column evidence of whether a semantic hint actually bound to any value.
32///
33/// A hint expresses user domain knowledge about a column; this records how much
34/// of the column that knowledge matched. `matched_values` counts the values of
35/// the kind the hint expects (numeric for `positive`, date-parseable for
36/// `temporal`, every non-null value for `identifier`, which coerces the column).
37///
38/// `exact` distinguishes a full-data count from a sampled one: a hint that
39/// matched nothing (`matched_values == 0`) is only *proven* inert when the count
40/// covers every row. A sampled zero is recorded but never treated as an error,
41/// because absence in a sample is not proof of absence in the data.
42#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
43pub struct SemanticHintBinding {
44    /// The hinted column this evidence is for.
45    pub column: String,
46    /// Which hint list named the column.
47    pub kind: SemanticHintKind,
48    /// Non-null values considered.
49    pub checked_values: usize,
50    /// Values that matched the hint's expected kind.
51    pub matched_values: usize,
52    /// Whether the counts cover every row (`true`) or a sample (`false`).
53    pub exact: bool,
54}
55
56impl SemanticHintBinding {
57    /// Whether this hint is *proven* inert: it was assessed over the full data
58    /// (`exact`), had values to consider (`checked_values > 0`), yet matched
59    /// none of them.
60    pub fn is_proven_inert(&self) -> bool {
61        self.exact && self.checked_values > 0 && self.matched_values == 0
62    }
63}
64
65/// User-supplied semantic hints that affect profiling and quality metrics.
66#[derive(Debug, Clone, Default, PartialEq, Eq)]
67#[non_exhaustive]
68pub struct SemanticHints {
69    pub positive_columns: Vec<String>,
70    pub identifier_columns: Vec<String>,
71    pub temporal_columns: Vec<String>,
72    /// Start/end column pairs declared by the caller, compared as given.
73    pub temporal_pairs: Vec<(String, String)>,
74}
75
76impl SemanticHints {
77    pub fn new(positive_columns: Vec<String>, identifier_columns: Vec<String>) -> Self {
78        Self {
79            positive_columns,
80            identifier_columns,
81            temporal_columns: Vec::new(),
82            temporal_pairs: Vec::new(),
83        }
84    }
85
86    pub fn with_temporal_columns(mut self, temporal_columns: Vec<String>) -> Self {
87        self.temporal_columns = temporal_columns;
88        self
89    }
90
91    /// Declare start/end column pairs for timeliness to compare as given,
92    /// such as `valid_from`/`valid_until`, which no role word pairs by name.
93    pub fn with_temporal_pairs(mut self, temporal_pairs: Vec<(String, String)>) -> Self {
94        self.temporal_pairs = temporal_pairs;
95        self
96    }
97
98    pub fn is_empty(&self) -> bool {
99        self.positive_columns.is_empty()
100            && self.identifier_columns.is_empty()
101            && self.temporal_columns.is_empty()
102            && self.temporal_pairs.is_empty()
103    }
104
105    pub fn is_identifier_column(&self, column_name: &str) -> bool {
106        self.identifier_columns
107            .iter()
108            .any(|candidate| candidate == column_name)
109    }
110
111    pub fn is_positive_column(&self, column_name: &str) -> bool {
112        self.positive_columns
113            .iter()
114            .any(|candidate| candidate == column_name)
115    }
116
117    pub fn is_temporal_column(&self, column_name: &str) -> bool {
118        self.temporal_columns
119            .iter()
120            .any(|candidate| candidate == column_name)
121    }
122
123    /// Each column named by a declared pair, once, in the order first named.
124    pub fn temporal_pair_columns(&self) -> Vec<&str> {
125        let mut columns: Vec<&str> = Vec::new();
126        for (start, end) in &self.temporal_pairs {
127            for name in [start.as_str(), end.as_str()] {
128                if !columns.contains(&name) {
129                    columns.push(name);
130                }
131            }
132        }
133        columns
134    }
135
136    /// Every `(kind, column_name)` pair once, preserving the first occurrence
137    /// within each hint list. A column may still carry different hint kinds.
138    pub fn iter(&self) -> impl Iterator<Item = (SemanticHintKind, &str)> {
139        let positive = self
140            .positive_columns
141            .iter()
142            .map(|c| (SemanticHintKind::Positive, c.as_str()));
143        let identifier = self
144            .identifier_columns
145            .iter()
146            .map(|c| (SemanticHintKind::Identifier, c.as_str()));
147        let temporal = self
148            .temporal_columns
149            .iter()
150            .map(|c| (SemanticHintKind::Temporal, c.as_str()));
151        let pairs = self
152            .temporal_pair_columns()
153            .into_iter()
154            .map(|c| (SemanticHintKind::TemporalPair, c));
155        let mut seen = std::collections::HashSet::new();
156        positive
157            .chain(identifier)
158            .chain(temporal)
159            .chain(pairs)
160            .filter(move |&(kind, column)| seen.insert((kind.field_name(), column)))
161    }
162
163    /// Fail loudly when a hint names a column that is not in the schema.
164    ///
165    /// A hint is the user's chosen alternative to overconfident inference, so a
166    /// hint that cannot bind must not vanish: a typo'd or stale column name is a
167    /// mistake the profiler should surface, not silently discard.
168    pub fn validate_names(&self, column_names: &[&str]) -> Result<(), DataProfilerError> {
169        // A column compared with itself can never be out of order, so such a
170        // pair would report a clean check that never happened.
171        if let Some((name, _)) = self.temporal_pairs.iter().find(|(start, end)| start == end) {
172            return Err(DataProfilerError::InvalidSemanticHint {
173                message: format!("temporal_pairs names '{name}' as both start and end"),
174                suggestion: "Pair a start column with a different end column.".to_string(),
175            });
176        }
177        // A pair declared in both orders is compared both ways, so one of the
178        // two reports every row whose dates differ as out of order.
179        if let Some((start, end)) = self.temporal_pairs.iter().find(|(start, end)| {
180            self.temporal_pairs
181                .iter()
182                .any(|(other_start, other_end)| other_start == end && other_end == start)
183        }) {
184            return Err(DataProfilerError::InvalidSemanticHint {
185                message: format!(
186                    "temporal_pairs names both ('{start}', '{end}') and ('{end}', '{start}')"
187                ),
188                suggestion: "Keep only the order with the start column first.".to_string(),
189            });
190        }
191        let mut unknown: Vec<(SemanticHintKind, &str)> = self
192            .iter()
193            .filter(|(_, name)| !column_names.contains(name))
194            .collect();
195        if unknown.is_empty() {
196            return Ok(());
197        }
198        // Stable, readable ordering regardless of hint-list order.
199        unknown.sort_unstable_by(|a, b| a.1.cmp(b.1).then(a.0.field_name().cmp(b.0.field_name())));
200
201        let details = unknown
202            .iter()
203            .map(|(kind, name)| format!("'{}' ({})", name, kind.field_name()))
204            .collect::<Vec<_>>()
205            .join(", ");
206        let mut available: Vec<&str> = column_names.to_vec();
207        available.sort_unstable();
208        let available = available
209            .iter()
210            .map(|c| format!("'{c}'"))
211            .collect::<Vec<_>>()
212            .join(", ");
213
214        Err(DataProfilerError::InvalidSemanticHint {
215            message: format!("semantic hint names not found in the data: {details}"),
216            suggestion: format!(
217                "Available columns: [{available}]. Check spelling and case, or remove the hint."
218            ),
219        })
220    }
221
222    /// Reject value-driven hints when the Quality metric pack did not run.
223    ///
224    /// Positive and temporal hints only affect quality calculators. Accepting
225    /// them without a quality assessment would make the configuration look
226    /// effective while producing neither metrics nor binding evidence.
227    pub fn validate_quality_usage(&self, quality_computed: bool) -> Result<(), DataProfilerError> {
228        if quality_computed
229            || (self.positive_columns.is_empty()
230                && self.temporal_columns.is_empty()
231                && self.temporal_pairs.is_empty())
232        {
233            return Ok(());
234        }
235
236        Err(DataProfilerError::InvalidSemanticHint {
237            message:
238                "positive_columns, temporal_columns and temporal_pairs require the Quality metric pack"
239                    .to_string(),
240            suggestion: "Include MetricPack::Quality (\"quality\" in Python), or remove the value-driven hint. Identifier hints remain usable without quality because they affect column typing."
241                .to_string(),
242        })
243    }
244
245    /// Fail loudly when a hint names a real column but bound to nothing.
246    ///
247    /// Only bindings that were assessed over the full data
248    /// ([`SemanticHintBinding::is_proven_inert`]) raise; sampled evidence is
249    /// carried in the report but never treated as proof of absence.
250    pub fn validate_bindings(
251        &self,
252        bindings: &[SemanticHintBinding],
253    ) -> Result<(), DataProfilerError> {
254        let inert: Vec<&SemanticHintBinding> =
255            bindings.iter().filter(|b| b.is_proven_inert()).collect();
256        if inert.is_empty() {
257            return Ok(());
258        }
259
260        let details = inert
261            .iter()
262            .map(|b| {
263                format!(
264                    "'{}' ({}): 0 of {} value(s) matched",
265                    b.column,
266                    b.kind.field_name(),
267                    b.checked_values
268                )
269            })
270            .collect::<Vec<_>>()
271            .join("; ");
272
273        Err(DataProfilerError::InvalidSemanticHint {
274            message: format!("semantic hint(s) bound to no matching value: {details}"),
275            suggestion:
276                "positive_columns need numeric values; temporal_columns and temporal_pairs \
277                 need date values. \
278                 Fix the column choice or remove the hint."
279                    .to_string(),
280        })
281    }
282}
dataprof_core/semantic.rs

dataprof_core/
semantic.rs
