1use std::collections::HashMap;
9
10use dataprof_core::{
11 AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, DataType, ExecutionMetadata,
12 QualityDimension, SemanticHintBinding, SemanticHintKind, SemanticHints,
13};
14use dataprof_metrics::{
15 MetricConfidence, MetricsCalculator, QualityAssessment, RowCompletenessSummary,
16 RowDuplicateSummary, analysis::metrics::BifurcatedResult, compute_value_hint_bindings,
17};
18
19use crate::{ProfileReport, QualityAnalysisStatus};
20
21pub struct ReportAssembler {
23 source: DataSource,
24 execution: ExecutionMetadata,
25 columns: Vec<ColumnProfile>,
26 quality_data: Option<HashMap<String, Vec<String>>>,
27 confidence: Option<MetricConfidence>,
28 skip: Option<QualityAnalysisStatus>,
31 requested_dimensions: Option<Vec<QualityDimension>>,
32 semantic_hints: SemanticHints,
33 exact_value_hint_bindings: Option<Vec<SemanticHintBinding>>,
34 row_duplicates: Option<RowDuplicateSummary>,
35 row_completeness: Option<RowCompletenessSummary>,
36 #[cfg(test)]
42 forced_quality_failure: Option<String>,
43}
44
45impl ReportAssembler {
46 pub fn new(source: DataSource, execution: ExecutionMetadata) -> Self {
48 Self {
49 source,
50 execution,
51 columns: Vec::new(),
52 quality_data: None,
53 confidence: None,
54 skip: None,
55 requested_dimensions: None,
56 semantic_hints: SemanticHints::default(),
57 exact_value_hint_bindings: None,
58 row_duplicates: None,
59 row_completeness: None,
60 #[cfg(test)]
61 forced_quality_failure: None,
62 }
63 }
64
65 #[cfg(test)]
68 fn force_quality_failure(mut self, message: &str) -> Self {
69 self.forced_quality_failure = Some(message.to_string());
70 self
71 }
72
73 pub fn columns(mut self, columns: Vec<ColumnProfile>) -> Self {
75 self.columns = columns;
76 self
77 }
78
79 pub fn with_quality_data(mut self, data: HashMap<String, Vec<String>>) -> Self {
81 self.quality_data = Some(data);
82 self
83 }
84
85 pub fn with_confidence(mut self, confidence: MetricConfidence) -> Self {
87 self.confidence = Some(confidence);
88 self
89 }
90
91 pub fn skip_quality(mut self) -> Self {
98 self.skip = Some(QualityAnalysisStatus::NotRequested);
99 self
100 }
101
102 pub fn skip_quality_no_data(mut self) -> Self {
112 if !matches!(self.skip, Some(QualityAnalysisStatus::NotRequested)) {
113 self.skip = Some(QualityAnalysisStatus::NoData);
114 }
115 self
116 }
117
118 pub fn with_requested_dimensions(mut self, dims: Vec<QualityDimension>) -> Self {
120 self.requested_dimensions = Some(dims);
121 self
122 }
123
124 pub fn with_semantic_hints(mut self, hints: SemanticHints) -> Self {
126 self.semantic_hints = hints;
127 self
128 }
129
130 pub fn with_analysis_options(mut self, options: &AnalysisOptions) -> Self {
139 self.skip = (!options.include_quality()).then_some(QualityAnalysisStatus::NotRequested);
140 self.semantic_hints = options.semantic_hints().clone();
141 self.requested_dimensions = options.quality_dimensions().map(<[_]>::to_vec);
142 if options.has_column_projection() && self.skip.is_none() {
143 let mut dimensions = self
149 .requested_dimensions
150 .take()
151 .unwrap_or_else(QualityDimension::all);
152 dimensions.retain(|dimension| {
153 !matches!(
154 dimension,
155 QualityDimension::Completeness | QualityDimension::Uniqueness
156 )
157 });
158 if dimensions.is_empty() {
159 self.skip = Some(QualityAnalysisStatus::WithheldByProjection);
160 }
161 self.requested_dimensions = Some(dimensions);
162 }
163 self
164 }
165
166 pub fn with_exact_value_hint_bindings(mut self, bindings: Vec<SemanticHintBinding>) -> Self {
171 self.exact_value_hint_bindings = Some(bindings);
172 self
173 }
174
175 pub fn with_row_duplicates(mut self, summary: Option<RowDuplicateSummary>) -> Self {
178 self.row_duplicates = summary;
179 self
180 }
181
182 pub fn with_row_completeness(mut self, summary: Option<RowCompletenessSummary>) -> Self {
187 self.row_completeness = summary;
188 self
189 }
190
191 pub fn build(mut self) -> ProfileReport {
199 self.withhold_container_values();
200 let (quality, status) = match &self.skip {
201 Some(reason) => (None, reason.clone()),
202 None => match &self.quality_data {
203 Some(data) => self.compute_quality(data),
204 None => (None, QualityAnalysisStatus::NoData),
205 },
206 };
207 let bindings = self.compute_hint_bindings();
208
209 ProfileReport::new(self.source, self.columns, self.execution, quality)
210 .with_quality_status(status)
211 .with_semantic_hint_bindings(bindings)
212 }
213
214 fn withhold_container_values(&mut self) {
223 let Some(data) = self.quality_data.as_mut() else {
224 return;
225 };
226 for column in &self.columns {
227 if column.data_type == DataType::Nested
228 && let Some(values) = data.get_mut(&column.name)
229 {
230 values.clear();
231 }
232 }
233 }
234
235 fn compute_hint_bindings(&self) -> Vec<SemanticHintBinding> {
243 if self.semantic_hints.is_empty() {
244 return Vec::new();
245 }
246
247 let mut bindings = Vec::new();
248 for column in &self.semantic_hints.identifier_columns {
249 if let Some(profile) = self.columns.iter().find(|c| &c.name == column) {
250 let checked = profile.total_count.saturating_sub(profile.null_count);
251 let matched = if profile.data_type == DataType::Identifier {
252 checked
253 } else {
254 0
255 };
256 bindings.push(SemanticHintBinding {
257 column: column.clone(),
258 kind: SemanticHintKind::Identifier,
259 checked_values: checked,
260 matched_values: matched,
261 exact: true,
262 });
263 }
264 }
265
266 if let Some(exact) = &self.exact_value_hint_bindings {
267 let full_coverage = self.execution.source_exhausted && !self.execution.sampling_applied;
268 bindings.extend(exact.iter().cloned().map(|mut binding| {
269 binding.exact &= full_coverage;
274 binding
275 }));
276 } else if let Some(data) = &self.quality_data {
277 let sample_size = data.values().map(|v| v.len()).max().unwrap_or(0);
278 let exact = !self.is_streaming_context(sample_size);
279 bindings.extend(compute_value_hint_bindings(
280 data,
281 &self.semantic_hints,
282 exact,
283 ));
284 }
285
286 bindings
287 }
288
289 fn compute_quality(
290 &self,
291 data: &HashMap<String, Vec<String>>,
292 ) -> (Option<QualityAssessment>, QualityAnalysisStatus) {
293 #[cfg(test)]
294 if let Some(message) = &self.forced_quality_failure {
295 return Self::failed(DataProfilerError::MetricsCalculationError {
296 message: message.clone(),
297 });
298 }
299
300 let sample_size = data.values().map(|v| v.len()).max().unwrap_or(0);
301 let is_streaming = self.is_streaming_context(sample_size);
302
303 if is_streaming {
304 self.compute_bifurcated_quality(data)
305 } else {
306 self.compute_uniform_quality(data)
307 }
308 }
309
310 fn failed(error: DataProfilerError) -> (Option<QualityAssessment>, QualityAnalysisStatus) {
312 log::warn!("Quality metrics calculation failed: {error}");
316 (
317 None,
318 QualityAnalysisStatus::Failed {
319 error: error.to_string(),
320 },
321 )
322 }
323
324 fn is_streaming_context(&self, sample_size: usize) -> bool {
325 self.execution.sampling_applied
326 || (sample_size > 0 && sample_size < self.execution.rows_processed)
327 }
328
329 fn compute_bifurcated_quality(
330 &self,
331 data: &HashMap<String, Vec<String>>,
332 ) -> (Option<QualityAssessment>, QualityAnalysisStatus) {
333 let calculator = MetricsCalculator::new().with_row_completeness(self.row_completeness);
334 match calculator.calculate_bifurcated_metrics_with_all_semantic_hints(
335 data,
336 &self.columns,
337 self.requested_dimensions.as_deref(),
338 &self.semantic_hints,
339 self.row_duplicates,
340 ) {
341 Ok(result) => {
342 let confidence = self
343 .confidence
344 .clone()
345 .unwrap_or_else(|| self.mixed_confidence(&result));
346 (
347 Some(
348 QualityAssessment::new(result.metrics, confidence)
349 .with_score_bounds(result.score_bounds),
350 ),
351 QualityAnalysisStatus::Computed,
352 )
353 }
354 Err(error) => Self::failed(error),
355 }
356 }
357
358 fn compute_uniform_quality(
359 &self,
360 data: &HashMap<String, Vec<String>>,
361 ) -> (Option<QualityAssessment>, QualityAnalysisStatus) {
362 let calculator = MetricsCalculator::new().with_row_completeness(self.row_completeness);
363 match calculator.calculate_comprehensive_metrics_with_all_semantic_hints(
364 data,
365 &self.columns,
366 self.requested_dimensions.as_deref(),
367 &self.semantic_hints,
368 self.row_duplicates,
369 ) {
370 Ok(metrics) => {
371 let confidence = self.confidence.clone().unwrap_or(MetricConfidence::Exact);
372 (
373 Some(QualityAssessment::new(metrics, confidence)),
374 QualityAnalysisStatus::Computed,
375 )
376 }
377 Err(error) => Self::failed(error),
378 }
379 }
380
381 fn mixed_confidence(&self, result: &BifurcatedResult) -> MetricConfidence {
382 MetricConfidence::Mixed {
383 exact_dimensions: result.exact_dimensions.clone(),
384 sampled_dimensions: result.sampled_dimensions.clone(),
385 sample_size: result.sample_size,
386 }
387 }
388}
389
390#[cfg(test)]
391mod tests {
392 use super::*;
393 use dataprof_core::{FileFormat, MetricPack, TruncationReason};
394
395 fn test_source() -> DataSource {
396 DataSource::File {
397 path: "test.csv".to_string(),
398 format: FileFormat::Csv,
399 size_bytes: 1024,
400 modified_at: None,
401 parquet_metadata: None,
402 }
403 }
404
405 #[test]
406 fn test_basic_report_assembly() {
407 let report =
408 ReportAssembler::new(test_source(), ExecutionMetadata::new(100, 3, 50)).build();
409
410 assert_eq!(report.execution.rows_processed, 100);
411 assert!(report.quality.is_none());
412 assert!(report.column_profiles.is_empty());
413 }
414
415 #[test]
416 fn test_skip_quality() {
417 let mut data = HashMap::new();
418 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
419
420 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
421 .with_quality_data(data)
422 .skip_quality()
423 .build();
424
425 assert!(report.quality.is_none());
426 }
427
428 #[test]
431 fn failed_quality_computation_is_not_a_skip() {
432 let mut data = HashMap::new();
433 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
434
435 let failed = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
436 .with_quality_data(data.clone())
437 .force_quality_failure("uniqueness accumulator disagreed with the row count")
438 .build();
439 let skipped = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
440 .with_quality_data(data)
441 .skip_quality()
442 .build();
443
444 assert!(failed.quality.is_none());
445 assert!(skipped.quality.is_none());
446 assert_eq!(
447 failed.quality_status,
448 QualityAnalysisStatus::Failed {
449 error: "Metrics calculation failed: uniqueness accumulator disagreed with the \
450 row count"
451 .to_string(),
452 }
453 );
454 assert_eq!(skipped.quality_status, QualityAnalysisStatus::NotRequested);
455 assert_ne!(failed.quality_status, skipped.quality_status);
456 }
457
458 #[test]
459 fn computed_quality_reports_computed() {
460 let mut data = HashMap::new();
461 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
462
463 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
464 .with_quality_data(data)
465 .build();
466
467 assert!(report.quality.is_some());
468 assert_eq!(report.quality_status, QualityAnalysisStatus::Computed);
469 }
470
471 #[test]
472 fn absent_quality_data_reports_no_data() {
473 let explicit = ReportAssembler::new(test_source(), ExecutionMetadata::new(0, 0, 1))
474 .skip_quality_no_data()
475 .build();
476 let implicit = ReportAssembler::new(test_source(), ExecutionMetadata::new(0, 0, 1)).build();
477
478 assert_eq!(explicit.quality_status, QualityAnalysisStatus::NoData);
479 assert_eq!(implicit.quality_status, QualityAnalysisStatus::NoData);
480 }
481
482 #[test]
483 fn empty_quality_data_is_analyzed() {
484 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(0, 0, 1))
485 .with_quality_data(HashMap::new())
486 .build();
487
488 assert_eq!(report.quality_status, QualityAnalysisStatus::Computed);
489 assert!(report.quality_score().is_none());
490 assert!(
491 report
492 .quality
493 .unwrap()
494 .metrics
495 .assessed_dimensions()
496 .is_empty()
497 );
498 }
499
500 #[test]
503 fn deselected_quality_survives_an_unavailable_sample() {
504 let deselected =
505 AnalysisOptions::default().with_metric_packs(Some(vec![MetricPack::Schema]));
506 let options_first = ReportAssembler::new(test_source(), ExecutionMetadata::new(0, 0, 1))
507 .with_analysis_options(&deselected)
508 .skip_quality_no_data()
509 .build();
510 let skip_first = ReportAssembler::new(test_source(), ExecutionMetadata::new(0, 0, 1))
511 .skip_quality_no_data()
512 .with_analysis_options(&deselected)
513 .build();
514
515 assert_eq!(
516 options_first.quality_status,
517 QualityAnalysisStatus::NotRequested
518 );
519 assert_eq!(
520 skip_first.quality_status,
521 QualityAnalysisStatus::NotRequested
522 );
523 }
524
525 #[test]
529 fn an_unavailable_sample_outranks_projection_withholding() {
530 let projected = AnalysisOptions::default()
531 .with_quality_dimensions(Some(vec![
532 QualityDimension::Completeness,
533 QualityDimension::Uniqueness,
534 ]))
535 .with_columns(Some(vec!["col".to_string()]));
536
537 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(0, 0, 1))
538 .with_analysis_options(&projected)
539 .skip_quality_no_data()
540 .build();
541
542 assert_eq!(report.quality_status, QualityAnalysisStatus::NoData);
543 }
544
545 #[test]
549 fn projection_withholding_every_dimension_is_not_a_skip() {
550 let mut data = HashMap::new();
551 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
552 let options = AnalysisOptions::default()
553 .with_quality_dimensions(Some(vec![
554 QualityDimension::Completeness,
555 QualityDimension::Uniqueness,
556 ]))
557 .with_columns(Some(vec!["col".to_string()]));
558
559 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
560 .with_quality_data(data)
561 .with_analysis_options(&options)
562 .build();
563
564 assert!(report.quality.is_none());
565 assert_eq!(
566 report.quality_status,
567 QualityAnalysisStatus::WithheldByProjection
568 );
569 }
570
571 #[test]
572 fn test_batch_produces_exact_confidence() {
573 let mut data = HashMap::new();
574 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
575
576 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
577 .with_quality_data(data)
578 .build();
579
580 assert!(report.quality.is_some());
581 let quality = report.quality.unwrap();
582 assert!(matches!(quality.confidence, MetricConfidence::Exact));
583 }
584
585 #[test]
586 fn test_streaming_produces_mixed_confidence() {
587 let mut data = HashMap::new();
588 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
589
590 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(1000, 1, 50))
591 .with_quality_data(data)
592 .build();
593
594 assert!(report.quality.is_some());
595 let quality = report.quality.unwrap();
596 match &quality.confidence {
597 MetricConfidence::Mixed {
598 exact_dimensions,
599 sampled_dimensions,
600 sample_size,
601 } => {
602 assert!(exact_dimensions.contains(&"completeness".to_string()));
603 assert!(!exact_dimensions.contains(&"key_uniqueness".to_string()));
606 assert!(sampled_dimensions.contains(&"consistency".to_string()));
607 assert!(sampled_dimensions.contains(&"accuracy".to_string()));
608 assert!(sampled_dimensions.contains(&"timeliness".to_string()));
609 assert!(sampled_dimensions.contains(&"duplicate_rows".to_string()));
610 assert_eq!(*sample_size, 2);
611 }
612 other => panic!("Expected Mixed confidence, got {:?}", other),
613 }
614 }
615
616 #[test]
617 fn test_streaming_exact_row_duplicates_have_exact_provenance() {
618 let data = HashMap::from([("col".to_string(), vec!["a".to_string(), "b".to_string()])]);
619
620 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(1000, 1, 50))
621 .with_quality_data(data)
622 .with_row_duplicates(Some(RowDuplicateSummary {
623 duplicate_rows: 25,
624 rows_checked: 1000,
625 approximate: false,
626 max_duplicate_rows: 25,
627 }))
628 .build();
629
630 let quality = report.quality.expect("quality assessment");
631 let uniqueness = quality.metrics.uniqueness.expect("uniqueness metrics");
632 assert_eq!(uniqueness.duplicate_rows, 25);
633 assert_eq!(uniqueness.rows_checked, 1000);
634 assert!(!uniqueness.duplicate_rows_approximate);
635 match quality.confidence {
636 MetricConfidence::Mixed {
637 exact_dimensions,
638 sampled_dimensions,
639 ..
640 } => {
641 assert!(exact_dimensions.contains(&"duplicate_rows".to_string()));
642 assert!(!sampled_dimensions.contains(&"duplicate_rows".to_string()));
643 }
644 other => panic!("Expected Mixed confidence, got {other:?}"),
645 }
646 }
647
648 #[test]
649 fn test_streaming_approximate_row_duplicates_have_sampled_provenance() {
650 let data = HashMap::from([("col".to_string(), vec!["a".to_string(), "b".to_string()])]);
651
652 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(20_000, 1, 50))
653 .with_quality_data(data)
654 .with_row_duplicates(Some(RowDuplicateSummary {
655 duplicate_rows: 500,
656 rows_checked: 20_000,
657 approximate: true,
658 max_duplicate_rows: 500,
659 }))
660 .build();
661
662 let quality = report.quality.expect("quality assessment");
663 let uniqueness = quality.metrics.uniqueness.expect("uniqueness metrics");
664 assert!(uniqueness.duplicate_rows_approximate);
665 match quality.confidence {
666 MetricConfidence::Mixed {
667 exact_dimensions,
668 sampled_dimensions,
669 ..
670 } => {
671 assert!(!exact_dimensions.contains(&"duplicate_rows".to_string()));
672 assert!(sampled_dimensions.contains(&"duplicate_rows".to_string()));
673 }
674 other => panic!("Expected Mixed confidence, got {other:?}"),
675 }
676 }
677
678 #[test]
679 fn test_sampling_applied_triggers_bifurcation() {
680 let mut data = HashMap::new();
681 data.insert("col".to_string(), vec!["a".to_string(), "b".to_string()]);
682
683 let execution = ExecutionMetadata::new(2, 1, 10).with_sampling(0.1);
684
685 let report = ReportAssembler::new(test_source(), execution)
686 .with_quality_data(data)
687 .build();
688
689 assert!(report.quality.is_some());
690 let quality = report.quality.unwrap();
691 assert!(matches!(quality.confidence, MetricConfidence::Mixed { .. }));
692 }
693
694 fn positive_binding() -> SemanticHintBinding {
695 SemanticHintBinding {
696 column: "col".to_string(),
697 kind: SemanticHintKind::Positive,
698 checked_values: 2,
699 matched_values: 0,
700 exact: true,
701 }
702 }
703
704 #[test]
705 fn exact_hint_binding_stays_exact_for_exhaustive_stream() {
706 let report = ReportAssembler::new(test_source(), ExecutionMetadata::new(2, 1, 10))
707 .with_semantic_hints(SemanticHints::new(vec!["col".to_string()], vec![]))
708 .with_exact_value_hint_bindings(vec![positive_binding()])
709 .build();
710
711 assert!(report.semantic_hint_bindings[0].exact);
712 }
713
714 #[test]
715 fn exact_hint_binding_is_downgraded_for_sampled_or_truncated_execution() {
716 let executions = [
717 ExecutionMetadata::new(2, 1, 10).with_sampling(0.5),
718 ExecutionMetadata::new(2, 1, 10).with_truncation(TruncationReason::MaxRows(2)),
719 ];
720
721 for execution in executions {
722 let report = ReportAssembler::new(test_source(), execution)
723 .with_semantic_hints(SemanticHints::new(vec!["col".to_string()], vec![]))
724 .with_exact_value_hint_bindings(vec![positive_binding()])
725 .build();
726
727 assert!(!report.semantic_hint_bindings[0].exact);
728 assert!(!report.semantic_hint_bindings[0].is_proven_inert());
729 }
730 }
731}