dataprof-runtime 0.10.0

Shared runtime helpers and composed report types for dataprof
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

//! Exact, bounded-memory semantic-hint binding counts for streaming engines.

use dataprof_core::{SemanticHintBinding, SemanticHintKind, SemanticHints};
use dataprof_metrics::{is_null_like_token, value_matches_hint};

#[derive(Debug, Clone)]
struct BindingCounter {
    binding: SemanticHintBinding,
}

/// Accumulates value-driven hint evidence while an engine scans the full data.
///
/// The number of configured hints bounds memory use; no cell values are kept.
#[derive(Debug, Clone, Default)]
pub struct ValueHintBindingAccumulator {
    counters: Vec<BindingCounter>,
}

impl ValueHintBindingAccumulator {
    pub fn new(hints: &SemanticHints) -> Self {
        let positive = hints.positive_columns.iter().map(|column| BindingCounter {
            binding: SemanticHintBinding {
                column: column.clone(),
                kind: SemanticHintKind::Positive,
                checked_values: 0,
                matched_values: 0,
                exact: true,
            },
        });
        let temporal = hints.temporal_columns.iter().map(|column| BindingCounter {
            binding: SemanticHintBinding {
                column: column.clone(),
                kind: SemanticHintKind::Temporal,
                checked_values: 0,
                matched_values: 0,
                exact: true,
            },
        });
        Self {
            counters: positive.chain(temporal).collect(),
        }
    }

    /// Observe one cell using the same null and match predicates as the quality
    /// calculators and their sample-based binding evidence.
    pub fn observe(&mut self, column: &str, value: &str) {
        if is_null_like_token(value.trim()) {
            return;
        }
        for counter in self
            .counters
            .iter_mut()
            .filter(|counter| counter.binding.column == column)
        {
            counter.binding.checked_values += 1;
            if value_matches_hint(value, counter.binding.kind) {
                counter.binding.matched_values += 1;
            }
        }
    }

    /// Return exact evidence for hints whose columns exist in the source.
    /// Unknown names remain the schema validator's responsibility.
    pub fn bindings<'a>(
        &self,
        column_names: impl IntoIterator<Item = &'a str>,
    ) -> Vec<SemanticHintBinding> {
        let names: std::collections::HashSet<&str> = column_names.into_iter().collect();
        self.counters
            .iter()
            .filter(|counter| names.contains(counter.binding.column.as_str()))
            .map(|counter| counter.binding.clone())
            .collect()
    }

    pub fn merge(&mut self, other: &Self) {
        for other_counter in &other.counters {
            if let Some(counter) = self.counters.iter_mut().find(|candidate| {
                candidate.binding.column == other_counter.binding.column
                    && candidate.binding.kind == other_counter.binding.kind
            }) {
                counter.binding.checked_values += other_counter.binding.checked_values;
                counter.binding.matched_values += other_counter.binding.matched_values;
            } else {
                self.counters.push(other_counter.clone());
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn accumulates_exact_counts_with_calculator_semantics() {
        let hints = SemanticHints::new(vec!["amount".to_string()], vec![])
            .with_temporal_columns(vec!["event".to_string()]);
        let mut accumulator = ValueHintBindingAccumulator::new(&hints);
        for value in ["1", " 2", "NULL"] {
            accumulator.observe("amount", value);
        }
        for value in ["2020-01-01", "not-a-date", " 2021-01-01"] {
            accumulator.observe("event", value);
        }

        let bindings = accumulator.bindings(["amount", "event"]);
        assert_eq!(bindings[0].checked_values, 2);
        assert_eq!(bindings[0].matched_values, 1);
        assert!(bindings[0].exact);
        assert_eq!(bindings[1].checked_values, 3);
        assert_eq!(bindings[1].matched_values, 1);
        assert!(bindings[1].exact);
    }

    #[test]
    fn omits_unknown_columns() {
        let hints = SemanticHints::new(vec!["missing".to_string()], vec![]);
        let accumulator = ValueHintBindingAccumulator::new(&hints);
        assert!(accumulator.bindings(["present"]).is_empty());
    }
}