1use std::collections::HashMap;
9
10use dataprof_core::{
11 AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, DataType, ExecutionMetadata,
12 QualityDimension, SemanticHintBinding, SemanticHintKind, SemanticHints,
13};
14use dataprof_metrics::{
15 MetricConfidence, MetricsCalculator, QualityAssessment, RowCompletenessSummary,
16 RowDuplicateSummary, analysis::metrics::BifurcatedResult, compute_value_hint_bindings,
17};
18
19use crate::{AnalysisSettings, ProfileReport, QualityAnalysisStatus};
20
21pub struct ReportAssembler {
23 source: DataSource,
24 execution: ExecutionMetadata,
25 columns: Vec<ColumnProfile>,
26 quality_data: Option<HashMap<String, Vec<String>>>,
27 confidence: Option<MetricConfidence>,
28 skip: Option<QualityAnalysisStatus>,
31 requested_dimensions: Option<Vec<QualityDimension>>,
32 semantic_hints: SemanticHints,
33 exact_value_hint_bindings: Option<Vec<SemanticHintBinding>>,
34 row_duplicates: Option<RowDuplicateSummary>,
35 row_completeness: Option<RowCompletenessSummary>,
36 analysis_settings: Option<AnalysisSettings>,
38 #[cfg(test)]
44 forced_quality_failure: Option<String>,
45}
46
47impl ReportAssembler {
48 pub fn new(source: DataSource, execution: ExecutionMetadata) -> Self {
50 Self {
51 source,
52 execution,
53 columns: Vec::new(),
54 quality_data: None,
55 confidence: None,
56 skip: None,
57 requested_dimensions: None,
58 semantic_hints: SemanticHints::default(),
59 exact_value_hint_bindings: None,
60 row_duplicates: None,
61 row_completeness: None,
62 analysis_settings: None,
63 #[cfg(test)]
64 forced_quality_failure: None,
65 }
66 }
67
68 #[cfg(test)]
71 fn force_quality_failure(mut self, message: &str) -> Self {
72 self.forced_quality_failure = Some(message.to_string());
73 self
74 }
75
76 pub fn columns(mut self, columns: Vec<ColumnProfile>) -> Self {
78 self.columns = columns;
79 self
80 }
81
82 pub fn with_quality_data(mut self, data: HashMap<String, Vec<String>>) -> Self {
84 self.quality_data = Some(data);
85 self
86 }
87
88 pub fn with_confidence(mut self, confidence: MetricConfidence) -> Self {
90 self.confidence = Some(confidence);
91 self
92 }
93
94 pub fn skip_quality(mut self) -> Self {
101 self.skip = Some(QualityAnalysisStatus::NotRequested);
102 self
103 }
104
105 pub fn skip_quality_no_data(mut self) -> Self {
115 if !matches!(self.skip, Some(QualityAnalysisStatus::NotRequested)) {
116 self.skip = Some(QualityAnalysisStatus::NoData);
117 }
118 self
119 }
120
121 pub fn with_requested_dimensions(mut self, dims: Vec<QualityDimension>) -> Self {
123 self.requested_dimensions = Some(dims);
124 self
125 }
126
127 pub fn with_semantic_hints(mut self, hints: SemanticHints) -> Self {
129 self.semantic_hints = hints;
130 self
131 }
132
133 pub fn with_analysis_options(mut self, options: &AnalysisOptions) -> Self {
142 self.skip = (!options.include_quality()).then_some(QualityAnalysisStatus::NotRequested);
143 self.semantic_hints = options.semantic_hints().clone();
144 self = self.record_analysis_settings(options);
145 self.requested_dimensions = options.quality_dimensions().map(<[_]>::to_vec);
146 if options.has_column_projection() && self.skip.is_none() {
147 let mut dimensions = self
153 .requested_dimensions
154 .take()
155 .unwrap_or_else(QualityDimension::all);
156 dimensions.retain(|dimension| {
157 !matches!(
158 dimension,
159 QualityDimension::Completeness | QualityDimension::Uniqueness
160 )
161 });
162 if dimensions.is_empty() {
163 self.skip = Some(QualityAnalysisStatus::WithheldByProjection);
164 }
165 self.requested_dimensions = Some(dimensions);
166 }
167 self
168 }
169
170 pub fn record_analysis_settings(mut self, options: &AnalysisOptions) -> Self {
177 self.analysis_settings = Some(AnalysisSettings::from_options(options));
178 self
179 }
180
181 pub fn with_exact_value_hint_bindings(mut self, bindings: Vec<SemanticHintBinding>) -> Self {
186 self.exact_value_hint_bindings = Some(bindings);
187 self
188 }
189
190 pub fn with_row_duplicates(mut self, summary: Option<RowDuplicateSummary>) -> Self {
193 self.row_duplicates = summary;
194 self
195 }
196
197 pub fn with_row_completeness(mut self, summary: Option<RowCompletenessSummary>) -> Self {
202 self.row_completeness = summary;
203 self
204 }
205
206 pub fn build(mut self) -> ProfileReport {
214 self.withhold_container_values();
215 let (quality, status) = match &self.skip {
216 Some(reason) => (None, reason.clone()),
217 None => match &self.quality_data {
218 Some(data) => self.compute_quality(data),
219 None => (None, QualityAnalysisStatus::NoData),
220 },
221 };
222 let bindings = self.compute_hint_bindings();
223
224 ProfileReport::new(self.source, self.columns, self.execution, quality)
225 .with_quality_status(status)
226 .with_semantic_hint_bindings(bindings)
227 .with_analysis_settings(self.analysis_settings)
228 }
229
230 fn withhold_container_values(&mut self) {
239 let Some(data) = self.quality_data.as_mut() else {
240 return;
241 };
242 for column in &self.columns {
243 if column.data_type == DataType::Nested
244 && let Some(values) = data.get_mut(&column.name)
245 {
246 values.clear();
247 }
248 }
249 }
250
251 fn compute_hint_bindings(&self) -> Vec<SemanticHintBinding> {
259 if self.semantic_hints.is_empty() {
260 return Vec::new();
261 }
262
263 let mut bindings = Vec::new();
264 for (_, column) in self
265 .semantic_hints
266 .iter()
267 .filter(|(kind, _)| *kind == SemanticHintKind::Identifier)
268 {
269 if let Some(profile) = self.columns.iter().find(|c| c.name == column) {
270 let checked = profile.total_count.saturating_sub(profile.null_count);
271 let matched = if profile.data_type == DataType::Identifier {
272 checked
273 } else {
274 0
275 };
276 bindings.push(SemanticHintBinding {
277 column: column.to_string(),
278 kind: SemanticHintKind::Identifier,
279 checked_values: checked,
280 matched_values: matched,
281 exact: true,
282 });
283 }
284 }
285
286 if let Some(exact) = &self.exact_value_hint_bindings {
287 let full_coverage = self.execution.source_exhausted && !self.execution.sampling_applied;
288 bindings.extend(exact.iter().cloned().map(|mut binding| {
289 binding.exact &= full_coverage;
294 binding
295 }));
296 } else if let Some(data) = &self.quality_data {
297 let sample_size = data.values().map(|v| v.len()).max().unwrap_or(0);
298 let exact = !self.is_streaming_context(sample_size);
299 bindings.extend(compute_value_hint_bindings(
300 data,
301 &self.semantic_hints,
302 exact,
303 ));
304 }
305
306 bindings
307 }
308
309 fn compute_quality(
310 &self,
311 data: &HashMap<String, Vec<String>>,
312 ) -> (Option<QualityAssessment>, QualityAnalysisStatus) {
313 #[cfg(test)]
314 if let Some(message) = &self.forced_quality_failure {
315 return Self::failed(DataProfilerError::MetricsCalculationError {
316 message: message.clone(),
317 });
318 }
319
320 let sample_size = data.values().map(|v| v.len()).max().unwrap_or(0);
321 let is_streaming = self.is_streaming_context(sample_size);
322
323 if is_streaming {
324 self.compute_bifurcated_quality(data)
325 } else {
326 self.compute_uniform_quality(data)
327 }
328 }
329
330 fn failed(error: DataProfilerError) -> (Option<QualityAssessment>, QualityAnalysisStatus) {
332 log::warn!("Quality metrics calculation failed: {error}");
336 (
337 None,
338 QualityAnalysisStatus::Failed {
339 error: error.to_string(),
340 },
341 )
342 }
343
344 fn is_streaming_context(&self, sample_size: usize) -> bool {
345 self.execution.sampling_applied
346 || (sample_size > 0 && sample_size < self.execution.rows_processed)
347 }
348
349 fn compute_bifurcated_quality(
350 &self,
351 data: &HashMap<String, Vec<String>>,
352 ) -> (Option<QualityAssessment>, QualityAnalysisStatus) {
353 let calculator = MetricsCalculator::new().with_row_completeness(self.row_completeness);
354 match calculator.calculate_bifurcated_metrics_with_hints(
355 data,
356 &self.columns,
357 self.requested_dimensions.as_deref(),
358 &self.semantic_hints,
359 self.row_duplicates,
360 ) {
361 Ok(result) => {
362 let confidence = self
363 .confidence
364 .clone()
365 .unwrap_or_else(|| self.mixed_confidence(&result));
366 (
367 Some(
368 QualityAssessment::new(result.metrics, confidence)
369 .with_score_bounds(result.score_bounds),
370 ),
371 QualityAnalysisStatus::Computed,
372 )
373 }
374 Err(error) => Self::failed(error),
375 }
376 }
377
378 fn compute_uniform_quality(
379 &self,
380 data: &HashMap<String, Vec<String>>,
381 ) -> (Option<QualityAssessment>, QualityAnalysisStatus) {
382 let calculator = MetricsCalculator::new().with_row_completeness(self.row_completeness);
383 match calculator.calculate_comprehensive_metrics_with_hints(
384 data,
385 &self.columns,
386 self.requested_dimensions.as_deref(),
387 &self.semantic_hints,
388 self.row_duplicates,
389 ) {
390 Ok(metrics) => {
391 let confidence = self.confidence.clone().unwrap_or(MetricConfidence::Exact);
392 (
393 Some(QualityAssessment::new(metrics, confidence)),
394 QualityAnalysisStatus::Computed,
395 )
396 }
397 Err(error) => Self::failed(error),
398 }
399 }
400
401 fn mixed_confidence(&self, result: &BifurcatedResult) -> MetricConfidence {
402 MetricConfidence::Mixed {
403 exact_dimensions: result.exact_dimensions.clone(),
404 sampled_dimensions: result.sampled_dimensions.clone(),
405 sample_size: result.sample_size,
406 }
407 }
408}
409
410#[cfg(test)]
411mod tests {
412 use super::*;
413 use dataprof_core::{FileFormat, MetricPack, TruncationReason};
414
415 fn test_source() -> DataSource {
416 DataSource::File {
417 path: "test.csv".to_string(),
418 format: FileFormat::Csv,
419 size_bytes: 1024,
420 modified_at: None,
421 parquet_metadata: None,
422 }
423 }
424
425 #[test]
426 fn test_basic_report_assembly() {
427 let report =
428 ReportAssembler::new(test_source(), ExecutionMetadata::new(100, 3, 50)).build();
429
430 assert_eq!(report.execution.rows_processed, 100);
431 assert!(report.quality.is_none());
432 assert!(report.column_profiles.is_empty());
433 }
434
435 #[test]
436 fn test_skip_quality() {
437 let mut data = HashMap::new();
438 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
439
440 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
441 .with_quality_data(data)
442 .skip_quality()
443 .build();
444
445 assert!(report.quality.is_none());
446 }
447
448 #[test]
451 fn failed_quality_computation_is_not_a_skip() {
452 let mut data = HashMap::new();
453 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
454
455 let failed = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
456 .with_quality_data(data.clone())
457 .force_quality_failure("uniqueness accumulator disagreed with the row count")
458 .build();
459 let skipped = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
460 .with_quality_data(data)
461 .skip_quality()
462 .build();
463
464 assert!(failed.quality.is_none());
465 assert!(skipped.quality.is_none());
466 assert_eq!(
467 failed.quality_status,
468 QualityAnalysisStatus::Failed {
469 error: "Metrics calculation failed: uniqueness accumulator disagreed with the \
470 row count"
471 .to_string(),
472 }
473 );
474 assert_eq!(skipped.quality_status, QualityAnalysisStatus::NotRequested);
475 assert_ne!(failed.quality_status, skipped.quality_status);
476 }
477
478 #[test]
479 fn computed_quality_reports_computed() {
480 let mut data = HashMap::new();
481 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
482
483 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
484 .with_quality_data(data)
485 .build();
486
487 assert!(report.quality.is_some());
488 assert_eq!(report.quality_status, QualityAnalysisStatus::Computed);
489 }
490
491 #[test]
492 fn absent_quality_data_reports_no_data() {
493 let explicit = ReportAssembler::new(test_source(), ExecutionMetadata::new(0, 0, 1))
494 .skip_quality_no_data()
495 .build();
496 let implicit = ReportAssembler::new(test_source(), ExecutionMetadata::new(0, 0, 1)).build();
497
498 assert_eq!(explicit.quality_status, QualityAnalysisStatus::NoData);
499 assert_eq!(implicit.quality_status, QualityAnalysisStatus::NoData);
500 }
501
502 #[test]
503 fn empty_quality_data_is_analyzed() {
504 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(0, 0, 1))
505 .with_quality_data(HashMap::new())
506 .build();
507
508 assert_eq!(report.quality_status, QualityAnalysisStatus::Computed);
509 assert!(report.quality_score().is_none());
510 assert!(
511 report
512 .quality
513 .unwrap()
514 .metrics
515 .assessed_dimensions()
516 .is_empty()
517 );
518 }
519
520 #[test]
523 fn deselected_quality_survives_an_unavailable_sample() {
524 let deselected =
525 AnalysisOptions::default().with_metric_packs(Some(vec![MetricPack::Schema]));
526 let options_first = ReportAssembler::new(test_source(), ExecutionMetadata::new(0, 0, 1))
527 .with_analysis_options(&deselected)
528 .skip_quality_no_data()
529 .build();
530 let skip_first = ReportAssembler::new(test_source(), ExecutionMetadata::new(0, 0, 1))
531 .skip_quality_no_data()
532 .with_analysis_options(&deselected)
533 .build();
534
535 assert_eq!(
536 options_first.quality_status,
537 QualityAnalysisStatus::NotRequested
538 );
539 assert_eq!(
540 skip_first.quality_status,
541 QualityAnalysisStatus::NotRequested
542 );
543 }
544
545 #[test]
549 fn an_unavailable_sample_outranks_projection_withholding() {
550 let projected = AnalysisOptions::default()
551 .with_quality_dimensions(Some(vec![
552 QualityDimension::Completeness,
553 QualityDimension::Uniqueness,
554 ]))
555 .with_columns(Some(vec!["col".to_string()]));
556
557 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(0, 0, 1))
558 .with_analysis_options(&projected)
559 .skip_quality_no_data()
560 .build();
561
562 assert_eq!(report.quality_status, QualityAnalysisStatus::NoData);
563 }
564
565 #[test]
569 fn projection_withholding_every_dimension_is_not_a_skip() {
570 let mut data = HashMap::new();
571 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
572 let options = AnalysisOptions::default()
573 .with_quality_dimensions(Some(vec![
574 QualityDimension::Completeness,
575 QualityDimension::Uniqueness,
576 ]))
577 .with_columns(Some(vec!["col".to_string()]));
578
579 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
580 .with_quality_data(data)
581 .with_analysis_options(&options)
582 .build();
583
584 assert!(report.quality.is_none());
585 assert_eq!(
586 report.quality_status,
587 QualityAnalysisStatus::WithheldByProjection
588 );
589 }
590
591 #[test]
592 fn test_batch_produces_exact_confidence() {
593 let mut data = HashMap::new();
594 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
595
596 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
597 .with_quality_data(data)
598 .build();
599
600 assert!(report.quality.is_some());
601 let quality = report.quality.unwrap();
602 assert!(matches!(quality.confidence, MetricConfidence::Exact));
603 }
604
605 #[test]
606 fn test_streaming_produces_mixed_confidence() {
607 let mut data = HashMap::new();
608 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
609
610 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(1000, 1, 50))
611 .with_quality_data(data)
612 .build();
613
614 assert!(report.quality.is_some());
615 let quality = report.quality.unwrap();
616 match &quality.confidence {
617 MetricConfidence::Mixed {
618 exact_dimensions,
619 sampled_dimensions,
620 sample_size,
621 } => {
622 assert!(exact_dimensions.contains(&"completeness".to_string()));
623 assert!(!exact_dimensions.contains(&"key_uniqueness".to_string()));
626 assert!(sampled_dimensions.contains(&"consistency".to_string()));
627 assert!(sampled_dimensions.contains(&"accuracy".to_string()));
628 assert!(sampled_dimensions.contains(&"timeliness".to_string()));
629 assert!(sampled_dimensions.contains(&"duplicate_rows".to_string()));
630 assert_eq!(*sample_size, 2);
631 }
632 other => panic!("Expected Mixed confidence, got {:?}", other),
633 }
634 }
635
636 #[test]
637 fn test_streaming_exact_row_duplicates_have_exact_provenance() {
638 let data = HashMap::from([("col".to_string(), vec!["a".to_string(), "b".to_string()])]);
639
640 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(1000, 1, 50))
641 .with_quality_data(data)
642 .with_row_duplicates(Some(RowDuplicateSummary {
643 duplicate_rows: 25,
644 rows_checked: 1000,
645 approximate: false,
646 max_duplicate_rows: 25,
647 }))
648 .build();
649
650 let quality = report.quality.expect("quality assessment");
651 let uniqueness = quality.metrics.uniqueness.expect("uniqueness metrics");
652 assert_eq!(uniqueness.duplicate_rows, 25);
653 assert_eq!(uniqueness.rows_checked, 1000);
654 assert!(!uniqueness.duplicate_rows_approximate);
655 match quality.confidence {
656 MetricConfidence::Mixed {
657 exact_dimensions,
658 sampled_dimensions,
659 ..
660 } => {
661 assert!(exact_dimensions.contains(&"duplicate_rows".to_string()));
662 assert!(!sampled_dimensions.contains(&"duplicate_rows".to_string()));
663 }
664 other => panic!("Expected Mixed confidence, got {other:?}"),
665 }
666 }
667
668 #[test]
669 fn test_streaming_approximate_row_duplicates_have_sampled_provenance() {
670 let data = HashMap::from([("col".to_string(), vec!["a".to_string(), "b".to_string()])]);
671
672 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(20_000, 1, 50))
673 .with_quality_data(data)
674 .with_row_duplicates(Some(RowDuplicateSummary {
675 duplicate_rows: 500,
676 rows_checked: 20_000,
677 approximate: true,
678 max_duplicate_rows: 500,
679 }))
680 .build();
681
682 let quality = report.quality.expect("quality assessment");
683 let uniqueness = quality.metrics.uniqueness.expect("uniqueness metrics");
684 assert!(uniqueness.duplicate_rows_approximate);
685 match quality.confidence {
686 MetricConfidence::Mixed {
687 exact_dimensions,
688 sampled_dimensions,
689 ..
690 } => {
691 assert!(!exact_dimensions.contains(&"duplicate_rows".to_string()));
692 assert!(sampled_dimensions.contains(&"duplicate_rows".to_string()));
693 }
694 other => panic!("Expected Mixed confidence, got {other:?}"),
695 }
696 }
697
698 #[test]
699 fn test_sampling_applied_triggers_bifurcation() {
700 let mut data = HashMap::new();
701 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
702
703 let execution = ExecutionMetadata::new(2, 1, 10).with_sampling(0.1);
704
705 let report = ReportAssembler::new(test_source(), execution)
706 .with_quality_data(data)
707 .build();
708
709 assert!(report.quality.is_some());
710 let quality = report.quality.unwrap();
711 assert!(matches!(quality.confidence, MetricConfidence::Mixed { .. }));
712 }
713
714 fn positive_binding() -> SemanticHintBinding {
715 SemanticHintBinding {
716 column: "col".to_string(),
717 kind: SemanticHintKind::Positive,
718 checked_values: 2,
719 matched_values: 0,
720 exact: true,
721 }
722 }
723
724 #[test]
725 fn exact_hint_binding_stays_exact_for_exhaustive_stream() {
726 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
727 .with_semantic_hints(SemanticHints::new(vec!["col".to_string()], vec![]))
728 .with_exact_value_hint_bindings(vec![positive_binding()])
729 .build();
730
731 assert!(report.semantic_hint_bindings[0].exact);
732 }
733
734 #[test]
735 fn exact_hint_binding_is_downgraded_for_sampled_or_truncated_execution() {
736 let executions = [
737 ExecutionMetadata::new(2, 1, 10).with_sampling(0.5),
738 ExecutionMetadata::new(2, 1, 10).with_truncation(TruncationReason::MaxRows(2)),
739 ];
740
741 for execution in executions {
742 let report = ReportAssembler::new(test_source(), execution)
743 .with_semantic_hints(SemanticHints::new(vec!["col".to_string()], vec![]))
744 .with_exact_value_hint_bindings(vec![positive_binding()])
745 .build();
746
747 assert!(!report.semantic_hint_bindings[0].exact);
748 assert!(!report.semantic_hint_bindings[0].is_proven_inert());
749 }
750 }
751}