1use serde_json::{Value, json};
2
3use super::QueryMetrics;
4
5#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
7pub enum MetricUnit {
8 Count,
10 Rows,
12 DeviceBytes,
14 HostBytes,
16 ParquetBytes,
18 Nanos,
20 Millis,
22 Boolean,
24 Label,
26}
27
28impl MetricUnit {
29 pub const fn as_str(self) -> &'static str {
31 match self {
32 Self::Count => "count",
33 Self::Rows => "rows",
34 Self::DeviceBytes => "device_bytes",
35 Self::HostBytes => "host_bytes",
36 Self::ParquetBytes => "parquet_bytes",
37 Self::Nanos => "nanos",
38 Self::Millis => "millis",
39 Self::Boolean => "boolean",
40 Self::Label => "label",
41 }
42 }
43}
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
47pub enum MetricAggregation {
48 Sum,
50 Peak,
52 Last,
54 RowCount,
56}
57
58impl MetricAggregation {
59 pub const fn as_str(self) -> &'static str {
61 match self {
62 Self::Sum => "sum",
63 Self::Peak => "peak",
64 Self::Last => "last",
65 Self::RowCount => "row_count",
66 }
67 }
68}
69
70#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
72pub enum MetricDirection {
73 LowerIsBetter,
75 HigherIsBetter,
77 Informational,
79}
80
81impl MetricDirection {
82 pub const fn as_str(self) -> &'static str {
84 match self {
85 Self::LowerIsBetter => "lower_is_better",
86 Self::HigherIsBetter => "higher_is_better",
87 Self::Informational => "informational",
88 }
89 }
90}
91
92#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
94pub struct MetricDescriptor {
95 pub name: &'static str,
97 pub unit: MetricUnit,
99 pub aggregation: MetricAggregation,
101 pub semantics: &'static str,
103 pub direction: Option<MetricDirection>,
105 pub healthy: Option<&'static str>,
107}
108
109impl MetricDescriptor {
110 pub fn to_json(self) -> Value {
112 json!({
113 "name": self.name,
114 "unit": self.unit.as_str(),
115 "aggregation": self.aggregation.as_str(),
116 "semantics": self.semantics,
117 "direction": self.direction.map(MetricDirection::as_str),
118 "healthy": self.healthy,
119 })
120 }
121}
122
123const fn metric(
124 name: &'static str,
125 unit: MetricUnit,
126 aggregation: MetricAggregation,
127 semantics: &'static str,
128 direction: Option<MetricDirection>,
129 healthy: Option<&'static str>,
130) -> MetricDescriptor {
131 MetricDescriptor { name, unit, aggregation, semantics, direction, healthy }
132}
133
134pub(crate) const QUERY_METRIC_DESCRIPTORS: &[MetricDescriptor] = &[
136 metric(
137 "input_batches",
138 MetricUnit::Count,
139 MetricAggregation::Sum,
140 "Input Arrow batches observed by native execution.",
141 None,
142 None,
143 ),
144 metric(
145 "input_rows",
146 MetricUnit::Rows,
147 MetricAggregation::Sum,
148 "Input rows observed by native execution.",
149 None,
150 None,
151 ),
152 metric(
153 "gpu_resident_input_handoffs",
154 MetricUnit::Count,
155 MetricAggregation::Sum,
156 "GPU-resident child outputs handed directly to native execution.",
157 Some(MetricDirection::HigherIsBetter),
158 None,
159 ),
160 metric(
161 "host_to_device_uploads",
162 MetricUnit::Count,
163 MetricAggregation::Sum,
164 "Host-to-device upload operations.",
165 Some(MetricDirection::LowerIsBetter),
166 None,
167 ),
168 metric(
169 "host_to_device_uploaded_rows",
170 MetricUnit::Rows,
171 MetricAggregation::Sum,
172 "Rows uploaded from host to device.",
173 Some(MetricDirection::LowerIsBetter),
174 None,
175 ),
176 metric(
177 "host_to_device_uploaded_bytes",
178 MetricUnit::HostBytes,
179 MetricAggregation::Sum,
180 "Bytes uploaded from host to device.",
181 Some(MetricDirection::LowerIsBetter),
182 None,
183 ),
184 metric(
185 "device_to_host_materializations",
186 MetricUnit::Count,
187 MetricAggregation::Sum,
188 "Device-to-host materialization operations.",
189 Some(MetricDirection::LowerIsBetter),
190 None,
191 ),
192 metric(
193 "device_to_host_materialized_rows",
194 MetricUnit::Rows,
195 MetricAggregation::Sum,
196 "Rows materialized from device to host.",
197 Some(MetricDirection::LowerIsBetter),
198 None,
199 ),
200 metric(
201 "device_to_host_materialized_bytes",
202 MetricUnit::DeviceBytes,
203 MetricAggregation::Sum,
204 "Bytes materialized from device to host.",
205 Some(MetricDirection::LowerIsBetter),
206 None,
207 ),
208 metric(
209 "device_to_device_copies",
210 MetricUnit::Count,
211 MetricAggregation::Sum,
212 "Device-to-device copy operations.",
213 Some(MetricDirection::LowerIsBetter),
214 None,
215 ),
216 metric(
217 "device_to_device_copied_bytes",
218 MetricUnit::DeviceBytes,
219 MetricAggregation::Sum,
220 "Bytes copied between device frames.",
221 Some(MetricDirection::LowerIsBetter),
222 None,
223 ),
224 metric(
225 "max_task_observed_peak_bytes",
226 MetricUnit::DeviceBytes,
227 MetricAggregation::Peak,
228 "Largest lower-level cuDF peak bytes reported by a task-like operation.",
229 Some(MetricDirection::LowerIsBetter),
230 None,
231 ),
232 metric(
233 "max_task_observed_peak_bytes_observed",
234 MetricUnit::Boolean,
235 MetricAggregation::Last,
236 "Whether max_task_observed_peak_bytes was reported by a lower layer.",
237 None,
238 None,
239 ),
240 metric(
241 "max_task_retained_bytes",
242 MetricUnit::DeviceBytes,
243 MetricAggregation::Peak,
244 "Largest retained byte count reported by a task-like lower-level cuDF operation.",
245 Some(MetricDirection::LowerIsBetter),
246 None,
247 ),
248 metric(
249 "max_task_retained_bytes_observed",
250 MetricUnit::Boolean,
251 MetricAggregation::Last,
252 "Whether max_task_retained_bytes was reported by a lower layer.",
253 None,
254 None,
255 ),
256 metric(
257 "native_graph_claim_calibration_rows",
258 MetricUnit::Count,
259 MetricAggregation::RowCount,
260 "Nested measured graph-execution peak rows.",
261 None,
262 None,
263 ),
264 metric(
265 "native_graph_runtime_rows",
266 MetricUnit::Count,
267 MetricAggregation::RowCount,
268 "Nested graph algorithm runtime evidence rows.",
269 None,
270 None,
271 ),
272 metric(
273 "native_task_operator_rows",
274 MetricUnit::Count,
275 MetricAggregation::RowCount,
276 "Nested per-operator task aggregate rows.",
277 None,
278 None,
279 ),
280 metric(
281 "native_task_runtime_rows",
282 MetricUnit::Count,
283 MetricAggregation::RowCount,
284 "Nested per-compiled-task runtime rows.",
285 None,
286 None,
287 ),
288 metric(
289 "native_plan_task_mapping_rows",
290 MetricUnit::Count,
291 MetricAggregation::RowCount,
292 "Nested native plan-node to compiled-task mapping rows.",
293 None,
294 None,
295 ),
296 metric(
297 "native_dynamic_filter_runtime_rows",
298 MetricUnit::Count,
299 MetricAggregation::RowCount,
300 "Nested dynamic-filter runtime evidence rows.",
301 None,
302 None,
303 ),
304 metric(
305 "native_source_read_runtime_rows",
306 MetricUnit::Count,
307 MetricAggregation::RowCount,
308 "Nested source-read runtime evidence rows.",
309 None,
310 None,
311 ),
312 metric(
313 "peak_device_memory_bytes",
314 MetricUnit::DeviceBytes,
315 MetricAggregation::Peak,
316 "Backward-compatible alias for resident_dataframe_peak_bytes.",
317 Some(MetricDirection::LowerIsBetter),
318 None,
319 ),
320 metric(
321 "resident_dataframe_peak_bytes",
322 MetricUnit::DeviceBytes,
323 MetricAggregation::Peak,
324 "Peak bytes retained by resident cudf DataFrames.",
325 Some(MetricDirection::LowerIsBetter),
326 None,
327 ),
328 metric(
329 "resident_dataframe_peak_bytes_observed",
330 MetricUnit::Boolean,
331 MetricAggregation::Last,
332 "Whether resident_dataframe_peak_bytes was observed.",
333 None,
334 None,
335 ),
336 metric(
337 "allocator_high_water_bytes",
338 MetricUnit::DeviceBytes,
339 MetricAggregation::Peak,
340 "cuDF allocator high-water bytes when reported by lower layers.",
341 Some(MetricDirection::LowerIsBetter),
342 None,
343 ),
344 metric(
345 "allocator_high_water_bytes_observed",
346 MetricUnit::Boolean,
347 MetricAggregation::Last,
348 "Whether allocator_high_water_bytes was observed.",
349 None,
350 None,
351 ),
352 metric(
353 "source_chunks_planned",
354 MetricUnit::Count,
355 MetricAggregation::Sum,
356 "Source chunks planned by native source execution.",
357 None,
358 None,
359 ),
360 metric(
361 "source_chunks_attempted",
362 MetricUnit::Count,
363 MetricAggregation::Sum,
364 "Source chunk attempts including retries.",
365 Some(MetricDirection::LowerIsBetter),
366 None,
367 ),
368 metric(
369 "source_chunks_committed",
370 MetricUnit::Count,
371 MetricAggregation::Sum,
372 "Source chunks committed successfully.",
373 None,
374 None,
375 ),
376 metric(
377 "source_chunks_retried",
378 MetricUnit::Count,
379 MetricAggregation::Sum,
380 "Source chunks retried.",
381 Some(MetricDirection::LowerIsBetter),
382 Some("0"),
383 ),
384 metric(
385 "source_read_prefetches_started",
386 MetricUnit::Count,
387 MetricAggregation::Sum,
388 "Source-read prefetches started.",
389 None,
390 None,
391 ),
392 metric(
393 "source_read_prefetches_completed",
394 MetricUnit::Count,
395 MetricAggregation::Sum,
396 "Source-read prefetches completed.",
397 None,
398 None,
399 ),
400 metric(
401 "source_read_prefetches_skipped",
402 MetricUnit::Count,
403 MetricAggregation::Sum,
404 "Source-read prefetches skipped.",
405 Some(MetricDirection::LowerIsBetter),
406 None,
407 ),
408 metric(
409 "source_read_prefetches_abandoned",
410 MetricUnit::Count,
411 MetricAggregation::Sum,
412 "Source-read prefetches abandoned.",
413 Some(MetricDirection::LowerIsBetter),
414 Some("0"),
415 ),
416 metric(
417 "row_groups_planned",
418 MetricUnit::Count,
419 MetricAggregation::Sum,
420 "Row groups planned across source chunks.",
421 None,
422 None,
423 ),
424 metric(
425 "row_groups_committed",
426 MetricUnit::Count,
427 MetricAggregation::Sum,
428 "Row groups committed across source chunks.",
429 None,
430 None,
431 ),
432 metric(
433 "partial_aggregate_count",
434 MetricUnit::Count,
435 MetricAggregation::Sum,
436 "Partial aggregate tasks executed.",
437 None,
438 None,
439 ),
440 metric(
441 "partial_aggregate_input_rows",
442 MetricUnit::Rows,
443 MetricAggregation::Sum,
444 "Rows consumed by partial aggregates.",
445 None,
446 None,
447 ),
448 metric(
449 "partial_aggregate_rows",
450 MetricUnit::Rows,
451 MetricAggregation::Sum,
452 "Rows produced by partial aggregates.",
453 None,
454 None,
455 ),
456 metric(
457 "partial_aggregate_retained_bytes",
458 MetricUnit::DeviceBytes,
459 MetricAggregation::Peak,
460 "Peak bytes retained by partial aggregate outputs.",
461 Some(MetricDirection::LowerIsBetter),
462 None,
463 ),
464 metric(
465 "final_merge_attempts",
466 MetricUnit::Count,
467 MetricAggregation::Sum,
468 "Final aggregate merge attempts.",
469 Some(MetricDirection::LowerIsBetter),
470 None,
471 ),
472 metric(
473 "final_merge_levels",
474 MetricUnit::Count,
475 MetricAggregation::Sum,
476 "Final aggregate merge levels.",
477 Some(MetricDirection::LowerIsBetter),
478 None,
479 ),
480 metric(
481 "final_merge_max_fan_in",
482 MetricUnit::Count,
483 MetricAggregation::Peak,
484 "Largest final merge fan-in.",
485 Some(MetricDirection::LowerIsBetter),
486 None,
487 ),
488 metric(
489 "compiled_pipeline_attempts",
490 MetricUnit::Count,
491 MetricAggregation::Sum,
492 "Compiled-pipeline execution attempts admitted before runtime.",
493 None,
494 None,
495 ),
496 metric(
497 "compiled_pipeline_tasks_executed",
498 MetricUnit::Count,
499 MetricAggregation::Sum,
500 "Compiled-pipeline tasks executed by the scheduler.",
501 None,
502 None,
503 ),
504 metric(
505 "compiled_pipeline_host_arrow_sinks",
506 MetricUnit::Count,
507 MetricAggregation::Sum,
508 "Host Arrow sink boundaries materialized by compiled pipelines.",
509 Some(MetricDirection::LowerIsBetter),
510 None,
511 ),
512 metric(
513 "parquet_files",
514 MetricUnit::Count,
515 MetricAggregation::Sum,
516 "Parquet files observed by source reads.",
517 None,
518 None,
519 ),
520 metric(
521 "parquet_bytes_known",
522 MetricUnit::ParquetBytes,
523 MetricAggregation::Sum,
524 "Known Parquet file bytes.",
525 None,
526 None,
527 ),
528 metric(
529 "row_groups_selected",
530 MetricUnit::Count,
531 MetricAggregation::Sum,
532 "Row groups selected by source reads.",
533 Some(MetricDirection::LowerIsBetter),
534 None,
535 ),
536 metric(
537 "read_columns",
538 MetricUnit::Count,
539 MetricAggregation::Sum,
540 "Physical read columns selected by source reads.",
541 Some(MetricDirection::LowerIsBetter),
542 None,
543 ),
544 metric(
545 "retry_count",
546 MetricUnit::Count,
547 MetricAggregation::Sum,
548 "Native execution retries.",
549 Some(MetricDirection::LowerIsBetter),
550 Some("0"),
551 ),
552 metric(
553 "retry_resource_exhausted_count",
554 MetricUnit::Count,
555 MetricAggregation::Sum,
556 "Retries caused by lower-level resource exhaustion.",
557 Some(MetricDirection::LowerIsBetter),
558 Some("0"),
559 ),
560 metric(
561 "dynamic_filter_produced_count",
562 MetricUnit::Count,
563 MetricAggregation::Sum,
564 "Dynamic-filter payloads produced.",
565 Some(MetricDirection::HigherIsBetter),
566 None,
567 ),
568 metric(
569 "dynamic_filter_skipped_count",
570 MetricUnit::Count,
571 MetricAggregation::Sum,
572 "Dynamic-filter routes skipped without a usable payload.",
573 Some(MetricDirection::LowerIsBetter),
574 None,
575 ),
576 metric(
577 "dynamic_filter_applied_count",
578 MetricUnit::Count,
579 MetricAggregation::Sum,
580 "Dynamic-filter payloads applied to runtime consumers.",
581 Some(MetricDirection::HigherIsBetter),
582 None,
583 ),
584 metric(
585 "dynamic_filter_row_groups_pruned",
586 MetricUnit::Count,
587 MetricAggregation::Sum,
588 "Row groups pruned by dynamic filters.",
589 Some(MetricDirection::HigherIsBetter),
590 None,
591 ),
592 metric(
593 "dynamic_filter_source_bytes_pruned",
594 MetricUnit::ParquetBytes,
595 MetricAggregation::Sum,
596 "Estimated source bytes pruned by dynamic filters.",
597 Some(MetricDirection::HigherIsBetter),
598 None,
599 ),
600 metric(
601 "dynamic_filter_reader_predicate_rows_before",
602 MetricUnit::Rows,
603 MetricAggregation::Sum,
604 "Source rows selected before dynamic reader predicates.",
605 None,
606 None,
607 ),
608 metric(
609 "dynamic_filter_reader_predicate_rows_after",
610 MetricUnit::Rows,
611 MetricAggregation::Sum,
612 "Source rows materialized after dynamic reader predicates.",
613 Some(MetricDirection::LowerIsBetter),
614 None,
615 ),
616 metric(
617 "dynamic_filter_reader_predicate_rows_filtered",
618 MetricUnit::Rows,
619 MetricAggregation::Sum,
620 "Source rows filtered by dynamic reader predicates.",
621 Some(MetricDirection::HigherIsBetter),
622 None,
623 ),
624 metric(
625 "dynamic_filter_post_decode_rows_before",
626 MetricUnit::Rows,
627 MetricAggregation::Sum,
628 "Rows before post-decode dynamic filters.",
629 None,
630 None,
631 ),
632 metric(
633 "dynamic_filter_post_decode_rows_after",
634 MetricUnit::Rows,
635 MetricAggregation::Sum,
636 "Rows after post-decode dynamic filters.",
637 Some(MetricDirection::LowerIsBetter),
638 None,
639 ),
640 metric(
641 "dynamic_filter_post_decode_rows_filtered",
642 MetricUnit::Rows,
643 MetricAggregation::Sum,
644 "Rows filtered by post-decode dynamic filters.",
645 Some(MetricDirection::HigherIsBetter),
646 None,
647 ),
648 metric(
649 "dynamic_filter_source_post_decode_rows_filtered",
650 MetricUnit::Rows,
651 MetricAggregation::Sum,
652 "Rows filtered by source-adjacent post-decode dynamic filters.",
653 Some(MetricDirection::HigherIsBetter),
654 None,
655 ),
656 metric(
657 "dynamic_filter_filter_post_decode_rows_filtered",
658 MetricUnit::Rows,
659 MetricAggregation::Sum,
660 "Rows filtered by filter-operator post-decode dynamic filters.",
661 Some(MetricDirection::HigherIsBetter),
662 None,
663 ),
664 metric(
665 "dynamic_filter_join_probe_post_decode_rows_filtered",
666 MetricUnit::Rows,
667 MetricAggregation::Sum,
668 "Rows filtered by hash-join-probe post-decode dynamic filters.",
669 Some(MetricDirection::HigherIsBetter),
670 None,
671 ),
672 metric(
673 "hash_join_build_count",
674 MetricUnit::Count,
675 MetricAggregation::Sum,
676 "Reusable cuDF hash-join artifacts built.",
677 None,
678 None,
679 ),
680 metric(
681 "filtered_join_build_count",
682 MetricUnit::Count,
683 MetricAggregation::Sum,
684 "Reusable cuDF filtered-join artifacts built.",
685 None,
686 None,
687 ),
688 metric(
689 "one_shot_join_count",
690 MetricUnit::Count,
691 MetricAggregation::Sum,
692 "One-shot keyed equality joins executed.",
693 None,
694 None,
695 ),
696 metric(
697 "hash_join_hash_table_bytes",
698 MetricUnit::DeviceBytes,
699 MetricAggregation::Sum,
700 "Device bytes retained by reusable hash-join build artifacts.",
701 Some(MetricDirection::LowerIsBetter),
702 None,
703 ),
704 metric(
705 "filtered_join_hash_table_bytes",
706 MetricUnit::DeviceBytes,
707 MetricAggregation::Sum,
708 "Device bytes retained by reusable filtered-join build artifacts.",
709 Some(MetricDirection::LowerIsBetter),
710 None,
711 ),
712 metric(
713 "filtered_join_probe_state_bytes",
714 MetricUnit::DeviceBytes,
715 MetricAggregation::Sum,
716 "Device bytes retained by filtered-join probe states.",
717 Some(MetricDirection::LowerIsBetter),
718 None,
719 ),
720 metric(
721 "mixed_join_size_data_bytes",
722 MetricUnit::DeviceBytes,
723 MetricAggregation::Sum,
724 "Device bytes retained by mixed join size-data states.",
725 Some(MetricDirection::LowerIsBetter),
726 None,
727 ),
728 metric(
729 "one_shot_join_index_bytes",
730 MetricUnit::DeviceBytes,
731 MetricAggregation::Sum,
732 "Device bytes retained by one-shot join index maps.",
733 Some(MetricDirection::LowerIsBetter),
734 None,
735 ),
736 metric(
737 "one_shot_join_materialization_bytes",
738 MetricUnit::DeviceBytes,
739 MetricAggregation::Sum,
740 "Reservation bytes requested for one-shot join output materialization.",
741 Some(MetricDirection::LowerIsBetter),
742 None,
743 ),
744 metric(
745 "one_shot_join_output_bytes",
746 MetricUnit::DeviceBytes,
747 MetricAggregation::Sum,
748 "Observed device bytes of one-shot join output frames.",
749 Some(MetricDirection::LowerIsBetter),
750 None,
751 ),
752 metric(
753 "join_gather_map_bytes",
754 MetricUnit::DeviceBytes,
755 MetricAggregation::Sum,
756 "Device bytes retained by join gather maps.",
757 Some(MetricDirection::LowerIsBetter),
758 None,
759 ),
760 metric(
761 "native_aggregate_strategy_rows",
762 MetricUnit::Count,
763 MetricAggregation::RowCount,
764 "Nested aggregate strategy decision rows.",
765 None,
766 None,
767 ),
768 metric(
769 "native_join_strategy_rows",
770 MetricUnit::Count,
771 MetricAggregation::RowCount,
772 "Nested join strategy decision rows.",
773 None,
774 None,
775 ),
776 metric(
777 "source_metadata_load_time",
778 MetricUnit::Nanos,
779 MetricAggregation::Sum,
780 "Time spent loading source metadata.",
781 Some(MetricDirection::LowerIsBetter),
782 None,
783 ),
784 metric(
785 "source_read_queue_wait_time",
786 MetricUnit::Nanos,
787 MetricAggregation::Sum,
788 "Time spent waiting in the source-read queue.",
789 Some(MetricDirection::LowerIsBetter),
790 None,
791 ),
792 metric(
793 "parquet_read_time",
794 MetricUnit::Nanos,
795 MetricAggregation::Sum,
796 "Time spent inside Parquet read calls.",
797 Some(MetricDirection::LowerIsBetter),
798 None,
799 ),
800 metric(
801 "source_read_time",
802 MetricUnit::Nanos,
803 MetricAggregation::Sum,
804 "Time spent in source-read orchestration.",
805 Some(MetricDirection::LowerIsBetter),
806 None,
807 ),
808 metric(
809 "operator_time",
810 MetricUnit::Nanos,
811 MetricAggregation::Sum,
812 "Time spent in non-source operators.",
813 Some(MetricDirection::LowerIsBetter),
814 None,
815 ),
816 metric(
817 "chunk_operator_time",
818 MetricUnit::Nanos,
819 MetricAggregation::Sum,
820 "Time spent in per-chunk operators.",
821 Some(MetricDirection::LowerIsBetter),
822 None,
823 ),
824 metric(
825 "final_merge_time",
826 MetricUnit::Nanos,
827 MetricAggregation::Sum,
828 "Time spent in final merge operators.",
829 Some(MetricDirection::LowerIsBetter),
830 None,
831 ),
832 metric(
833 "final_sort_time",
834 MetricUnit::Nanos,
835 MetricAggregation::Sum,
836 "Time spent in final sort operators.",
837 Some(MetricDirection::LowerIsBetter),
838 None,
839 ),
840 metric(
841 "materialization_time",
842 MetricUnit::Nanos,
843 MetricAggregation::Sum,
844 "Time spent materializing final outputs.",
845 Some(MetricDirection::LowerIsBetter),
846 None,
847 ),
848];
849
850impl QueryMetrics {
851 pub fn descriptors() -> &'static [MetricDescriptor] {
853 QUERY_METRIC_DESCRIPTORS
854 }
855}