Skip to main content

dataprof_core/
analysis_options.rs

1//! The analysis selection every parser and engine must honour.
2//!
3//! Metric packs, quality dimensions, locale, and semantic hints are all
4//! *requests about what to compute*, and the report they produce has to be the
5//! same on every input path. Passing them as loose parameters made that easy to
6//! get wrong: a parser that took dimensions and hints but not packs or locale
7//! compiled fine and silently computed work the caller had deselected.
8//! [`AnalysisOptions`] bundles them so a path either carries the whole selection
9//! or does not compile.
10
11use std::collections::HashSet;
12
13use chrono::NaiveDate;
14
15use crate::errors::DataProfilerError;
16use crate::locale::Locale;
17use crate::quality::{MetricPack, QualityDimension};
18use crate::semantic::SemanticHints;
19
20/// What to analyze, and how, for a single profiling run.
21///
22/// Construct with [`AnalysisOptions::default`] (analyze everything) and narrow
23/// with the builder methods.
24#[derive(Debug, Clone, Default, PartialEq, Eq)]
25pub struct AnalysisOptions {
26    columns: Option<Vec<String>>,
27    metric_packs: Option<Vec<MetricPack>>,
28    quality_dimensions: Option<Vec<QualityDimension>>,
29    locale: Option<Locale>,
30    semantic_hints: SemanticHints,
31    reference_date: Option<NaiveDate>,
32    duplicate_rows_exact_limit: Option<usize>,
33}
34
35impl AnalysisOptions {
36    /// Select top-level columns to profile. `None` (the default) means every column.
37    ///
38    /// Selection is by name and reports remain in source-schema order. An empty
39    /// vector requests a schema with no profiled columns.
40    pub fn with_columns(mut self, columns: Option<Vec<String>>) -> Self {
41        self.columns = columns;
42        self
43    }
44
45    /// Select the metric packs to compute. `None` (the default) means all.
46    pub fn with_metric_packs(mut self, packs: Option<Vec<MetricPack>>) -> Self {
47        self.metric_packs = packs;
48        self
49    }
50
51    /// Select the quality dimensions to assess. `None` (the default) means all.
52    pub fn with_quality_dimensions(mut self, dimensions: Option<Vec<QualityDimension>>) -> Self {
53        self.quality_dimensions = dimensions;
54        self
55    }
56
57    /// Set the locale used to rank detected patterns.
58    ///
59    /// Parse a user-supplied tag with [`Locale::parse_optional`] first, so an
60    /// unrecognised tag is rejected where it is written rather than silently
61    /// suppressing every locale-specific pattern.
62    pub fn with_locale(mut self, locale: Option<Locale>) -> Self {
63        self.locale = locale;
64        self
65    }
66
67    /// Set the user's semantic hints.
68    pub fn with_semantic_hints(mut self, hints: SemanticHints) -> Self {
69        self.semantic_hints = hints;
70        self
71    }
72
73    /// Fix the date timeliness measures against. `None` (the default) reads
74    /// today's UTC date when the run starts, so an unchanged file's timeliness
75    /// moves as the calendar does; a fixed date makes it reproducible (#903).
76    /// The report records which one the score used.
77    pub fn with_reference_date(mut self, reference_date: Option<NaiveDate>) -> Self {
78        self.reference_date = reference_date;
79        self
80    }
81
82    /// Count duplicate rows exactly up to `limit` distinct rows (#1292).
83    ///
84    /// `None` (the default) keeps the built-in limit of one million distinct
85    /// rows, past which the count is an estimate and the memory budget may
86    /// end the exact count earlier. A limit set here is a row count, so the
87    /// verdict does not depend on the machine: the row tracker keeps its exact
88    /// set up to the limit whatever `memory_limit_mb` says, at 10 to 20 bytes
89    /// per distinct row, and that set is not counted against the budget.
90    pub fn with_duplicate_rows_exact_limit(mut self, limit: Option<usize>) -> Self {
91        self.duplicate_rows_exact_limit = limit;
92        self
93    }
94
95    /// The packs to compute, with an empty quality-dimension selection folded in.
96    ///
97    /// Resolved on read rather than in the setters so the outcome does not
98    /// depend on the order the selections were made in.
99    pub fn effective_metric_packs(&self) -> Option<Vec<MetricPack>> {
100        MetricPack::resolve_with_dimensions(
101            self.metric_packs.as_deref(),
102            self.quality_dimensions.as_deref(),
103        )
104    }
105
106    /// Every pack the run computes, in [`MetricPack::all`] order: the
107    /// effective selection with the schema pack, which is always computed,
108    /// made explicit. Two requests that compute the same packs return the
109    /// same list, whatever order or repetition they were given in.
110    pub fn selected_metric_packs(&self) -> Vec<MetricPack> {
111        let effective = self.effective_metric_packs();
112        MetricPack::all()
113            .into_iter()
114            .filter(|pack| {
115                *pack == MetricPack::Schema
116                    || effective
117                        .as_deref()
118                        .is_none_or(|selected| selected.contains(pack))
119            })
120            .collect()
121    }
122
123    /// Whether per-column statistics should be computed.
124    pub fn include_statistics(&self) -> bool {
125        MetricPack::include_statistics(self.effective_metric_packs().as_deref())
126    }
127
128    /// Whether pattern detection should run.
129    pub fn include_patterns(&self) -> bool {
130        MetricPack::include_patterns(self.effective_metric_packs().as_deref())
131    }
132
133    /// Whether quality metrics should be computed.
134    ///
135    /// `false` means the report carries no quality at all — absent, not an
136    /// empty assessment — because nothing was analyzed.
137    pub fn include_quality(&self) -> bool {
138        MetricPack::include_quality(self.effective_metric_packs().as_deref())
139    }
140
141    /// The locale used to rank detected patterns.
142    ///
143    /// A locale only ranks patterns, so it has no effect of its own when
144    /// [`include_patterns`](Self::include_patterns) is false and detection never
145    /// runs.
146    pub fn locale(&self) -> Option<Locale> {
147        self.locale
148    }
149
150    /// The fixed timeliness reference date, or `None` to read today's.
151    pub fn reference_date(&self) -> Option<NaiveDate> {
152        self.reference_date
153    }
154
155    /// The duplicate-row exact limit the caller set, or `None` for the default.
156    pub fn duplicate_rows_exact_limit(&self) -> Option<usize> {
157        self.duplicate_rows_exact_limit
158    }
159
160    /// The requested quality dimensions, if the caller narrowed them.
161    pub fn quality_dimensions(&self) -> Option<&[QualityDimension]> {
162        self.quality_dimensions.as_deref()
163    }
164
165    /// The user's semantic hints.
166    pub fn semantic_hints(&self) -> &SemanticHints {
167        &self.semantic_hints
168    }
169
170    /// The requested top-level column names, or `None` when every column is selected.
171    pub fn columns(&self) -> Option<&[String]> {
172        self.columns.as_deref()
173    }
174
175    /// Whether the caller selected a subset of the source columns.
176    pub fn has_column_projection(&self) -> bool {
177        self.columns.is_some()
178    }
179
180    /// Resolve the selected columns against a source schema.
181    ///
182    /// Returned indices follow source order, not request order, so every input
183    /// path preserves the same stable report ordering. Unknown and duplicate
184    /// requests fail rather than producing a plausible partial profile.
185    pub fn column_indices(
186        &self,
187        available: &[String],
188    ) -> Result<Option<Vec<usize>>, DataProfilerError> {
189        let Some(requested) = &self.columns else {
190            return Ok(None);
191        };
192
193        let mut seen = HashSet::with_capacity(requested.len());
194        let duplicates = requested
195            .iter()
196            .filter(|name| !seen.insert(name.as_str()))
197            .cloned()
198            .collect::<Vec<_>>();
199        if !duplicates.is_empty() {
200            return Err(DataProfilerError::invalid_config(
201                &format!(
202                    "column projection contains duplicate name(s): {}",
203                    duplicates.join(", ")
204                ),
205                "List each projected column once.",
206            ));
207        }
208
209        let available_set = available.iter().map(String::as_str).collect::<HashSet<_>>();
210        let missing = requested
211            .iter()
212            .filter(|name| !available_set.contains(name.as_str()))
213            .cloned()
214            .collect::<Vec<_>>();
215        if !missing.is_empty() {
216            return Err(DataProfilerError::invalid_config(
217                &format!(
218                    "column projection names column(s) absent from the source: {}",
219                    missing.join(", ")
220                ),
221                "Choose column names from the source schema.",
222            ));
223        }
224
225        Ok(Some(
226            available
227                .iter()
228                .enumerate()
229                .filter_map(|(index, name)| seen.contains(name.as_str()).then_some(index))
230                .collect(),
231        ))
232    }
233}
234
235#[cfg(test)]
236mod tests {
237    use super::*;
238
239    #[test]
240    fn default_analyzes_everything() {
241        let options = AnalysisOptions::default();
242        assert!(options.include_statistics());
243        assert!(options.include_patterns());
244        assert!(options.include_quality());
245        assert_eq!(options.locale(), None);
246        assert_eq!(options.quality_dimensions(), None);
247        assert_eq!(options.columns(), None);
248        assert!(!options.has_column_projection());
249    }
250
251    #[test]
252    fn schema_only_deselects_every_other_pack() {
253        let options = AnalysisOptions::default().with_metric_packs(Some(vec![MetricPack::Schema]));
254        assert!(!options.include_statistics());
255        assert!(!options.include_patterns());
256        assert!(!options.include_quality());
257    }
258
259    #[test]
260    fn empty_dimension_selection_removes_the_quality_pack() {
261        let options = AnalysisOptions::default().with_quality_dimensions(Some(vec![]));
262        assert!(!options.include_quality());
263        // Deselecting quality says nothing about the other packs.
264        assert!(options.include_statistics());
265        assert!(options.include_patterns());
266    }
267
268    #[test]
269    fn resolution_does_not_depend_on_setter_order() {
270        let packs = vec![MetricPack::Schema, MetricPack::Quality];
271        let dims_first = AnalysisOptions::default()
272            .with_quality_dimensions(Some(vec![]))
273            .with_metric_packs(Some(packs.clone()));
274        let packs_first = AnalysisOptions::default()
275            .with_metric_packs(Some(packs))
276            .with_quality_dimensions(Some(vec![]));
277        assert_eq!(
278            dims_first.effective_metric_packs(),
279            packs_first.effective_metric_packs()
280        );
281        assert!(!dims_first.include_quality());
282    }
283
284    #[test]
285    fn selected_packs_list_what_runs_once_in_a_fixed_order() {
286        use MetricPack::{Patterns, Quality, Schema, Statistics};
287        let selected = |options: AnalysisOptions| options.selected_metric_packs();
288        assert_eq!(
289            selected(AnalysisOptions::default()),
290            vec![Schema, Statistics, Patterns, Quality]
291        );
292        assert_eq!(
293            selected(
294                AnalysisOptions::default()
295                    .with_metric_packs(Some(vec![Quality, Patterns, Quality]))
296            ),
297            vec![Schema, Patterns, Quality]
298        );
299        assert_eq!(
300            selected(AnalysisOptions::default().with_metric_packs(Some(vec![]))),
301            vec![Schema]
302        );
303        assert_eq!(
304            selected(AnalysisOptions::default().with_quality_dimensions(Some(vec![]))),
305            vec![Schema, Statistics, Patterns]
306        );
307    }
308
309    #[test]
310    fn a_locale_alone_does_not_re_enable_pattern_detection() {
311        let options = AnalysisOptions::default()
312            .with_locale(Some(Locale::It))
313            .with_metric_packs(Some(vec![MetricPack::Schema]));
314        assert_eq!(options.locale(), Some(Locale::It));
315        assert!(!options.include_patterns());
316    }
317
318    #[test]
319    fn column_projection_resolves_in_source_order() {
320        let options = AnalysisOptions::default()
321            .with_columns(Some(vec!["city".to_string(), "id".to_string()]));
322        let available = vec!["id".to_string(), "name".to_string(), "city".to_string()];
323
324        assert_eq!(
325            options.column_indices(&available).unwrap(),
326            Some(vec![0, 2])
327        );
328        assert!(options.has_column_projection());
329    }
330
331    #[test]
332    fn column_projection_rejects_unknown_and_duplicate_names() {
333        let available = vec!["id".to_string(), "city".to_string()];
334        let unknown = AnalysisOptions::default().with_columns(Some(vec!["name".to_string()]));
335        assert!(unknown.column_indices(&available).is_err());
336
337        let duplicate =
338            AnalysisOptions::default().with_columns(Some(vec!["id".to_string(), "id".to_string()]));
339        assert!(duplicate.column_indices(&available).is_err());
340    }
341}