1use std::any::Any;
33use std::borrow::Cow;
34use std::fmt::{self, Debug, Display};
35use std::sync::Arc;
36
37use chrono::{DateTime, Utc};
38use datafusion_common::format::{MetricCategory, MetricType};
39use datafusion_physical_expr_common::metrics::{
40 Count, CustomMetricValue, Gauge, MetricValue, MetricsSet, PruningMetrics,
41 RatioMergeStrategy, RatioMetrics, Time, Timestamp,
42};
43use datafusion_physical_expr_common::metrics::{Label, Metric};
44use stabby::string::String as SString;
45use stabby::vec::Vec as SVec;
46
47use crate::ffi_option::FFI_Option;
48
49#[repr(C)]
51#[derive(Debug, Clone)]
52pub struct FFI_MetricsSet {
53 pub metrics: SVec<FFI_Metric>,
54}
55
56#[repr(C)]
58#[derive(Debug, Clone)]
59pub struct FFI_Metric {
60 pub value: FFI_MetricValue,
61 pub labels: SVec<FFI_Label>,
62 pub partition: FFI_Option<usize>,
63 pub metric_type: FFI_MetricType,
64 pub metric_category: FFI_Option<FFI_MetricCategory>,
65}
66
67#[repr(C)]
69#[derive(Debug, Clone)]
70pub struct FFI_Label {
71 pub name: SString,
72 pub value: SString,
73}
74
75#[expect(non_camel_case_types)]
77#[repr(u8)]
78#[derive(Debug, Clone, Copy, PartialEq, Eq)]
79pub enum FFI_MetricType {
80 Summary,
81 Dev,
82}
83
84#[expect(non_camel_case_types)]
86#[repr(u8)]
87#[derive(Debug, Clone, Copy, PartialEq, Eq)]
88pub enum FFI_MetricCategory {
89 Rows,
90 Bytes,
91 Timing,
92 Uncategorized,
93}
94
95#[repr(C)]
98#[derive(Debug, Clone, Copy, PartialEq, Eq)]
99pub struct FFI_PruningMetrics {
100 pub pruned: u64,
101 pub matched: u64,
102 pub fully_matched: u64,
103}
104
105#[expect(non_camel_case_types)]
107#[expect(
108 clippy::enum_variant_names,
109 reason = "match RatioMergeStrategy variants"
110)]
111#[repr(u8)]
112#[derive(Debug, Clone, Copy, PartialEq, Eq)]
113pub enum FFI_RatioMergeStrategy {
114 AddPartAddTotal,
115 AddPartSetTotal,
116 SetPartAddTotal,
117}
118
119#[repr(C)]
122#[derive(Debug, Clone, Copy, PartialEq, Eq)]
123pub struct FFI_RatioMetrics {
124 pub part: u64,
125 pub total: u64,
126 pub merge_strategy: FFI_RatioMergeStrategy,
127 pub display_raw_values: bool,
128}
129
130#[repr(C, u8)]
132#[derive(Debug, Clone)]
133pub enum FFI_MetricValue {
134 OutputRows(u64),
135 ElapsedComputeNs(u64),
136 SpillCount(u64),
137 SpilledBytes(u64),
138 OutputBytes(u64),
139 OutputBatches(u64),
140 SpilledRows(u64),
141 CurrentMemoryUsage(u64),
142 Count {
143 name: SString,
144 count: u64,
145 },
146 Gauge {
147 name: SString,
148 gauge: u64,
149 },
150 Time {
151 name: SString,
152 time_ns: u64,
153 },
154 StartTimestampNsUTC(FFI_Option<i64>),
155 EndTimestampNsUTC(FFI_Option<i64>),
156 PruningMetrics {
157 name: SString,
158 pruning_metrics: FFI_PruningMetrics,
159 },
160 Ratio {
161 name: SString,
162 ratio_metrics: FFI_RatioMetrics,
163 },
164 Custom {
169 name: SString,
170 display: SString,
171 as_usize_value: u64,
172 },
173}
174
175impl From<&MetricsSet> for FFI_MetricsSet {
180 fn from(set: &MetricsSet) -> Self {
181 Self {
182 metrics: set.iter().map(|m| FFI_Metric::from(m.as_ref())).collect(),
183 }
184 }
185}
186
187impl From<FFI_MetricsSet> for MetricsSet {
188 fn from(set: FFI_MetricsSet) -> Self {
189 let mut out = MetricsSet::new();
190 for ffi_metric in set.metrics {
191 out.push(Arc::new(Metric::from(ffi_metric)));
192 }
193 out
194 }
195}
196
197impl From<&Metric> for FFI_Metric {
202 fn from(m: &Metric) -> Self {
203 Self {
204 value: FFI_MetricValue::from(m.value()),
205 labels: m.labels().iter().map(FFI_Label::from).collect(),
206 partition: m.partition().into(),
207 metric_type: m.metric_type().into(),
208 metric_category: m.metric_category().map(FFI_MetricCategory::from).into(),
209 }
210 }
211}
212
213impl From<FFI_Metric> for Metric {
214 fn from(m: FFI_Metric) -> Self {
215 let labels: Vec<Label> = m.labels.into_iter().map(Label::from).collect();
216 let partition: Option<usize> = m.partition.into();
217 let category: Option<FFI_MetricCategory> = m.metric_category.into();
218 let mut metric = Metric::new_with_labels(m.value.into(), partition, labels)
219 .with_type(m.metric_type.into());
220 if let Some(c) = category {
221 metric = metric.with_category(c.into());
222 }
223 metric
224 }
225}
226
227impl From<&Label> for FFI_Label {
232 fn from(l: &Label) -> Self {
233 Self {
234 name: SString::from(l.name()),
235 value: SString::from(l.value()),
236 }
237 }
238}
239
240impl From<FFI_Label> for Label {
241 fn from(l: FFI_Label) -> Self {
242 let name: String = l.name.into();
243 let value: String = l.value.into();
244 Label::new(name, value)
245 }
246}
247
248impl From<MetricType> for FFI_MetricType {
253 fn from(t: MetricType) -> Self {
254 match t {
255 MetricType::Summary => Self::Summary,
256 MetricType::Dev => Self::Dev,
257 }
258 }
259}
260
261impl From<FFI_MetricType> for MetricType {
262 fn from(t: FFI_MetricType) -> Self {
263 match t {
264 FFI_MetricType::Summary => Self::Summary,
265 FFI_MetricType::Dev => Self::Dev,
266 }
267 }
268}
269
270impl From<MetricCategory> for FFI_MetricCategory {
275 fn from(c: MetricCategory) -> Self {
276 match c {
277 MetricCategory::Rows => Self::Rows,
278 MetricCategory::Bytes => Self::Bytes,
279 MetricCategory::Timing => Self::Timing,
280 MetricCategory::Uncategorized => Self::Uncategorized,
281 }
282 }
283}
284
285impl From<FFI_MetricCategory> for MetricCategory {
286 fn from(c: FFI_MetricCategory) -> Self {
287 match c {
288 FFI_MetricCategory::Rows => Self::Rows,
289 FFI_MetricCategory::Bytes => Self::Bytes,
290 FFI_MetricCategory::Timing => Self::Timing,
291 FFI_MetricCategory::Uncategorized => Self::Uncategorized,
292 }
293 }
294}
295
296impl From<&RatioMergeStrategy> for FFI_RatioMergeStrategy {
301 fn from(s: &RatioMergeStrategy) -> Self {
302 match s {
303 RatioMergeStrategy::AddPartAddTotal => Self::AddPartAddTotal,
304 RatioMergeStrategy::AddPartSetTotal => Self::AddPartSetTotal,
305 RatioMergeStrategy::SetPartAddTotal => Self::SetPartAddTotal,
306 }
307 }
308}
309
310impl From<FFI_RatioMergeStrategy> for RatioMergeStrategy {
311 fn from(s: FFI_RatioMergeStrategy) -> Self {
312 match s {
313 FFI_RatioMergeStrategy::AddPartAddTotal => Self::AddPartAddTotal,
314 FFI_RatioMergeStrategy::AddPartSetTotal => Self::AddPartSetTotal,
315 FFI_RatioMergeStrategy::SetPartAddTotal => Self::SetPartAddTotal,
316 }
317 }
318}
319
320impl From<&PruningMetrics> for FFI_PruningMetrics {
325 fn from(p: &PruningMetrics) -> Self {
326 Self {
327 pruned: p.pruned() as u64,
328 matched: p.matched() as u64,
329 fully_matched: p.fully_matched() as u64,
330 }
331 }
332}
333
334impl From<FFI_PruningMetrics> for PruningMetrics {
335 fn from(p: FFI_PruningMetrics) -> Self {
336 let out = PruningMetrics::new();
337 out.add_pruned(p.pruned as usize);
338 out.add_matched(p.matched as usize);
339 out.add_fully_matched(p.fully_matched as usize);
340 out
341 }
342}
343
344impl From<&RatioMetrics> for FFI_RatioMetrics {
349 fn from(r: &RatioMetrics) -> Self {
350 Self {
351 part: r.part() as u64,
352 total: r.total() as u64,
353 merge_strategy: r.merge_strategy().into(),
354 display_raw_values: r.display_raw_values(),
355 }
356 }
357}
358
359impl From<FFI_RatioMetrics> for RatioMetrics {
360 fn from(r: FFI_RatioMetrics) -> Self {
361 let out = RatioMetrics::new()
362 .with_merge_strategy(r.merge_strategy.into())
363 .with_display_raw_values(r.display_raw_values);
364 out.set_part(r.part as usize);
365 out.set_total(r.total as usize);
366 out
367 }
368}
369
370fn timestamp_to_ffi(ts: &Timestamp) -> FFI_Option<i64> {
375 ts.value().and_then(|dt| dt.timestamp_nanos_opt()).into()
376}
377
378fn timestamp_from_ffi(nanos: FFI_Option<i64>) -> Timestamp {
379 let ts = Timestamp::new();
380 if let Some(n) = nanos.into_option() {
381 ts.set(DateTime::<Utc>::from_timestamp_nanos(n));
382 }
383 ts
384}
385
386fn count_from_value(v: u64) -> Count {
387 let c = Count::new();
388 c.add(v as usize);
389 c
390}
391
392fn gauge_from_value(v: u64) -> Gauge {
393 let g = Gauge::new();
394 g.add(v as usize);
395 g
396}
397
398fn time_from_nanos(nanos: u64) -> Time {
399 let t = Time::new();
400 if nanos != 0 {
401 t.add_duration(std::time::Duration::from_nanos(nanos));
403 }
404 t
405}
406
407impl From<&MetricValue> for FFI_MetricValue {
408 fn from(v: &MetricValue) -> Self {
409 match v {
410 MetricValue::OutputRows(c) => Self::OutputRows(c.value() as u64),
411 MetricValue::ElapsedCompute(t) => Self::ElapsedComputeNs(t.value() as u64),
412 MetricValue::SpillCount(c) => Self::SpillCount(c.value() as u64),
413 MetricValue::SpilledBytes(c) => Self::SpilledBytes(c.value() as u64),
414 MetricValue::OutputBytes(c) => Self::OutputBytes(c.value() as u64),
415 MetricValue::OutputBatches(c) => Self::OutputBatches(c.value() as u64),
416 MetricValue::SpilledRows(c) => Self::SpilledRows(c.value() as u64),
417 MetricValue::CurrentMemoryUsage(g) => {
418 Self::CurrentMemoryUsage(g.value() as u64)
419 }
420 MetricValue::Count { name, count } => Self::Count {
421 name: SString::from(name.as_ref()),
422 count: count.value() as u64,
423 },
424 MetricValue::Gauge { name, gauge } => Self::Gauge {
425 name: SString::from(name.as_ref()),
426 gauge: gauge.value() as u64,
427 },
428 MetricValue::Time { name, time } => Self::Time {
429 name: SString::from(name.as_ref()),
430 time_ns: time.value() as u64,
431 },
432 MetricValue::StartTimestamp(ts) => {
433 Self::StartTimestampNsUTC(timestamp_to_ffi(ts))
434 }
435 MetricValue::EndTimestamp(ts) => {
436 Self::EndTimestampNsUTC(timestamp_to_ffi(ts))
437 }
438 MetricValue::PruningMetrics {
439 name,
440 pruning_metrics,
441 } => Self::PruningMetrics {
442 name: SString::from(name.as_ref()),
443 pruning_metrics: pruning_metrics.into(),
444 },
445 MetricValue::Ratio {
446 name,
447 ratio_metrics,
448 } => Self::Ratio {
449 name: SString::from(name.as_ref()),
450 ratio_metrics: ratio_metrics.into(),
451 },
452 MetricValue::Custom { name, value } => Self::Custom {
453 name: SString::from(name.as_ref()),
454 display: SString::from(value.to_string().as_str()),
455 as_usize_value: value.as_usize() as u64,
456 },
457 }
458 }
459}
460
461impl From<FFI_MetricValue> for MetricValue {
462 fn from(v: FFI_MetricValue) -> Self {
463 match v {
464 FFI_MetricValue::OutputRows(n) => Self::OutputRows(count_from_value(n)),
465 FFI_MetricValue::ElapsedComputeNs(n) => {
466 Self::ElapsedCompute(time_from_nanos(n))
467 }
468 FFI_MetricValue::SpillCount(n) => Self::SpillCount(count_from_value(n)),
469 FFI_MetricValue::SpilledBytes(n) => Self::SpilledBytes(count_from_value(n)),
470 FFI_MetricValue::OutputBytes(n) => Self::OutputBytes(count_from_value(n)),
471 FFI_MetricValue::OutputBatches(n) => Self::OutputBatches(count_from_value(n)),
472 FFI_MetricValue::SpilledRows(n) => Self::SpilledRows(count_from_value(n)),
473 FFI_MetricValue::CurrentMemoryUsage(n) => {
474 Self::CurrentMemoryUsage(gauge_from_value(n))
475 }
476 FFI_MetricValue::Count { name, count } => Self::Count {
477 name: Cow::Owned(name.into()),
478 count: count_from_value(count),
479 },
480 FFI_MetricValue::Gauge { name, gauge } => Self::Gauge {
481 name: Cow::Owned(name.into()),
482 gauge: gauge_from_value(gauge),
483 },
484 FFI_MetricValue::Time { name, time_ns } => Self::Time {
485 name: Cow::Owned(name.into()),
486 time: time_from_nanos(time_ns),
487 },
488 FFI_MetricValue::StartTimestampNsUTC(nanos) => {
489 Self::StartTimestamp(timestamp_from_ffi(nanos))
490 }
491 FFI_MetricValue::EndTimestampNsUTC(nanos) => {
492 Self::EndTimestamp(timestamp_from_ffi(nanos))
493 }
494 FFI_MetricValue::PruningMetrics {
495 name,
496 pruning_metrics,
497 } => Self::PruningMetrics {
498 name: Cow::Owned(name.into()),
499 pruning_metrics: pruning_metrics.into(),
500 },
501 FFI_MetricValue::Ratio {
502 name,
503 ratio_metrics,
504 } => Self::Ratio {
505 name: Cow::Owned(name.into()),
506 ratio_metrics: ratio_metrics.into(),
507 },
508 FFI_MetricValue::Custom {
509 name,
510 display,
511 as_usize_value,
512 } => Self::Custom {
513 name: Cow::Owned(name.into()),
514 value: Arc::new(FfiCustomMetricValue {
515 display: display.into(),
516 as_usize_value: as_usize_value as usize,
517 }),
518 },
519 }
520 }
521}
522
523#[derive(Debug)]
533pub struct FfiCustomMetricValue {
534 display: String,
535 as_usize_value: usize,
536}
537
538impl Display for FfiCustomMetricValue {
539 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
540 f.write_str(&self.display)
541 }
542}
543
544impl CustomMetricValue for FfiCustomMetricValue {
545 fn new_empty(&self) -> Arc<dyn CustomMetricValue> {
546 Arc::new(FfiCustomMetricValue {
547 display: String::new(),
548 as_usize_value: 0,
549 })
550 }
551
552 fn aggregate(&self, _other: Arc<dyn CustomMetricValue>) {
553 }
556
557 fn as_any(&self) -> &dyn Any {
558 self
559 }
560
561 fn as_usize(&self) -> usize {
562 self.as_usize_value
563 }
564
565 fn is_eq(&self, other: &Arc<dyn CustomMetricValue>) -> bool {
566 other
567 .as_any()
568 .downcast_ref::<Self>()
569 .map(|o| o.display == self.display && o.as_usize_value == self.as_usize_value)
570 .unwrap_or(false)
571 }
572}
573
574#[cfg(test)]
575mod tests {
576 use super::*;
577 use datafusion_physical_expr_common::metrics::Label;
578
579 fn assert_value_roundtrip(value: MetricValue) -> MetricValue {
585 let ffi: FFI_MetricValue = (&value).into();
586 let back: MetricValue = ffi.into();
587 assert_eq!(value, back, "round-trip mismatch for {value:?}");
588 back
589 }
590
591 #[test]
592 fn roundtrip_named_variants() {
593 let c = Count::new();
594 c.add(7);
595 assert_value_roundtrip(MetricValue::OutputRows(c.clone()));
596 assert_value_roundtrip(MetricValue::SpillCount(c.clone()));
597 assert_value_roundtrip(MetricValue::SpilledBytes(c.clone()));
598 assert_value_roundtrip(MetricValue::OutputBytes(c.clone()));
599 assert_value_roundtrip(MetricValue::OutputBatches(c.clone()));
600 assert_value_roundtrip(MetricValue::SpilledRows(c.clone()));
601
602 let g = Gauge::new();
603 g.add(123);
604 assert_value_roundtrip(MetricValue::CurrentMemoryUsage(g));
605
606 let t = Time::new();
607 t.add_duration(std::time::Duration::from_nanos(456));
608 assert_value_roundtrip(MetricValue::ElapsedCompute(t));
609 }
610
611 #[test]
612 fn roundtrip_keyed_count_gauge_time() {
613 let count = Count::new();
614 count.add(11);
615 assert_value_roundtrip(MetricValue::Count {
616 name: Cow::Borrowed("custom_count"),
617 count,
618 });
619
620 let gauge = Gauge::new();
621 gauge.add(22);
622 assert_value_roundtrip(MetricValue::Gauge {
623 name: Cow::Borrowed("custom_gauge"),
624 gauge,
625 });
626
627 let time = Time::new();
628 time.add_duration(std::time::Duration::from_nanos(33));
629 assert_value_roundtrip(MetricValue::Time {
630 name: Cow::Borrowed("custom_time"),
631 time,
632 });
633 }
634
635 #[test]
636 fn roundtrip_timestamps() {
637 let unset = Timestamp::new();
638 assert_value_roundtrip(MetricValue::StartTimestamp(unset.clone()));
639 assert_value_roundtrip(MetricValue::EndTimestamp(unset));
640
641 let set = Timestamp::new();
642 set.set(DateTime::<Utc>::from_timestamp_nanos(
643 1_700_000_000_000_000_000,
644 ));
645 assert_value_roundtrip(MetricValue::StartTimestamp(set.clone()));
646 assert_value_roundtrip(MetricValue::EndTimestamp(set));
647 }
648
649 #[test]
650 fn roundtrip_pruning_and_ratio() {
651 let pruning = PruningMetrics::new();
652 pruning.add_pruned(3);
653 pruning.add_matched(5);
654 pruning.add_fully_matched(2);
655 let back = assert_value_roundtrip(MetricValue::PruningMetrics {
656 name: Cow::Borrowed("file_prune"),
657 pruning_metrics: pruning,
658 });
659 match back {
660 MetricValue::PruningMetrics {
661 name,
662 pruning_metrics,
663 } => {
664 assert_eq!(name.as_ref(), "file_prune");
665 assert_eq!(pruning_metrics.pruned(), 3);
666 assert_eq!(pruning_metrics.matched(), 5);
667 assert_eq!(pruning_metrics.fully_matched(), 2);
668 }
669 other => panic!("expected PruningMetrics, got {other:?}"),
670 }
671
672 let ratio = RatioMetrics::new()
673 .with_merge_strategy(RatioMergeStrategy::SetPartAddTotal)
674 .with_display_raw_values(false);
675 ratio.set_part(20);
676 ratio.set_total(100);
677 let back = assert_value_roundtrip(MetricValue::Ratio {
680 name: Cow::Borrowed("selectivity"),
681 ratio_metrics: ratio,
682 });
683 match back {
684 MetricValue::Ratio {
685 name,
686 ratio_metrics,
687 } => {
688 assert_eq!(name.as_ref(), "selectivity");
689 assert_eq!(ratio_metrics.part(), 20);
690 assert_eq!(ratio_metrics.total(), 100);
691 assert!(matches!(
692 ratio_metrics.merge_strategy(),
693 RatioMergeStrategy::SetPartAddTotal
694 ));
695 assert!(!ratio_metrics.display_raw_values());
696 }
697 other => panic!("expected Ratio, got {other:?}"),
698 }
699 }
700
701 #[test]
702 fn custom_metric_value_stringifies() {
703 #[derive(Debug)]
704 struct DummyCustom;
705 impl Display for DummyCustom {
706 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
707 write!(f, "dummy:42")
708 }
709 }
710 impl CustomMetricValue for DummyCustom {
711 fn new_empty(&self) -> Arc<dyn CustomMetricValue> {
712 Arc::new(DummyCustom)
713 }
714 fn aggregate(&self, _other: Arc<dyn CustomMetricValue>) {}
715 fn as_any(&self) -> &dyn Any {
716 self
717 }
718 fn as_usize(&self) -> usize {
719 42
720 }
721 fn is_eq(&self, other: &Arc<dyn CustomMetricValue>) -> bool {
722 other.as_any().downcast_ref::<Self>().is_some()
723 }
724 }
725
726 let original = MetricValue::Custom {
727 name: Cow::Borrowed("dummy"),
728 value: Arc::new(DummyCustom),
729 };
730 let ffi: FFI_MetricValue = (&original).into();
731 let back: MetricValue = ffi.into();
732
733 match back {
734 MetricValue::Custom { name, value } => {
735 assert_eq!(name.as_ref(), "dummy");
736 assert_eq!(value.to_string(), "dummy:42");
737 assert_eq!(value.as_usize(), 42);
738 }
739 other => panic!("expected Custom, got {other:?}"),
740 }
741 }
742
743 #[test]
744 fn roundtrip_full_metric_with_labels_partition_type_category() {
745 let count = Count::new();
746 count.add(99);
747 let metric = Metric::new_with_labels(
748 MetricValue::OutputRows(count),
749 Some(3),
750 vec![Label::new("file", "a.parquet")],
751 )
752 .with_type(MetricType::Summary)
753 .with_category(MetricCategory::Rows);
754
755 let ffi: FFI_Metric = (&metric).into();
756 let back: Metric = ffi.into();
757
758 assert_eq!(back.value(), metric.value());
759 assert_eq!(back.partition(), Some(3));
760 assert_eq!(back.labels().len(), 1);
761 assert_eq!(back.labels()[0].name(), "file");
762 assert_eq!(back.labels()[0].value(), "a.parquet");
763 assert_eq!(back.metric_type(), MetricType::Summary);
764 assert_eq!(back.metric_category(), Some(MetricCategory::Rows));
765 }
766
767 #[test]
768 fn roundtrip_metrics_set() {
769 let mut set = MetricsSet::new();
770 let c = Count::new();
771 c.add(1);
772 set.push(Arc::new(Metric::new(MetricValue::OutputRows(c), Some(0))));
773 let c2 = Count::new();
774 c2.add(2);
775 set.push(Arc::new(Metric::new(MetricValue::OutputRows(c2), Some(1))));
776
777 let ffi: FFI_MetricsSet = (&set).into();
778 let back: MetricsSet = ffi.into();
779
780 assert_eq!(back.output_rows(), Some(3));
782
783 let metrics: Vec<_> = back.iter().collect();
787 assert_eq!(metrics.len(), 2);
788 assert_eq!(metrics[0].partition(), Some(0));
789 assert_eq!(
790 metrics[0].value(),
791 &MetricValue::OutputRows(count_from_value(1))
792 );
793 assert_eq!(metrics[1].partition(), Some(1));
794 assert_eq!(
795 metrics[1].value(),
796 &MetricValue::OutputRows(count_from_value(2))
797 );
798 }
799}