datafusion_tracing/lib.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17//
18// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc.
19
20//! DataFusion Tracing is an extension for [Apache DataFusion](https://datafusion.apache.org/) that helps you monitor and debug queries. It uses [`tracing`](https://docs.rs/tracing/latest/tracing/) and [OpenTelemetry](https://opentelemetry.io/) to gather DataFusion metrics, trace execution steps, and preview partial query results.
21//!
22//! **Note:** This is not an official Apache Software Foundation release.
23//!
24//! # Overview
25//!
26//! When you run queries with DataFusion Tracing enabled, it automatically adds tracing around execution steps, records all native DataFusion metrics such as execution time and output row count, lets you preview partial results for easier debugging, and integrates with OpenTelemetry for distributed tracing. This makes it simpler to understand and improve query performance.
27//!
28//! ## See it in action
29//!
30//! Here's what DataFusion Tracing can look like in practice:
31//!
32//! <details>
33//! <summary>Jaeger UI</summary>
34//!
35//! 
36//! </details>
37//!
38//! <details>
39//! <summary>DataDog UI</summary>
40//!
41//! 
42//! </details>
43//!
44//! # Getting Started
45//!
46//! ## Installation
47//!
48//! Include DataFusion Tracing in your project's `Cargo.toml`:
49//!
50//! ```toml
51//! [dependencies]
52//! datafusion = "49.0.0"
53//! datafusion-tracing = "49.0.0"
54//! ```
55//!
56//! ## Quick Start Example
57//!
58//! ```rust
59//! use datafusion::{
60//! arrow::{array::RecordBatch, util::pretty::pretty_format_batches},
61//! error::Result,
62//! execution::SessionStateBuilder,
63//! prelude::*,
64//! };
65//! use datafusion_tracing::{
66//! instrument_with_info_spans, pretty_format_compact_batch, InstrumentationOptions,
67//! };
68//! use std::sync::Arc;
69//! use tracing::field;
70//!
71//! #[tokio::main]
72//! async fn main() -> Result<()> {
73//! // Initialize tracing subscriber as usual
74//! // (See examples/otlp.rs for a complete example).
75//!
76//! // Set up tracing options (you can customize these).
77//! let options = InstrumentationOptions::builder()
78//! .record_metrics(true)
79//! .preview_limit(5)
80//! .preview_fn(Arc::new(|batch: &RecordBatch| {
81//! pretty_format_compact_batch(batch, 64, 3, 10).map(|fmt| fmt.to_string())
82//! }))
83//! .add_custom_field("env", "production")
84//! .add_custom_field("region", "us-west")
85//! .build();
86//!
87//! let instrument_rule = instrument_with_info_spans!(
88//! options: options,
89//! env = field::Empty,
90//! region = field::Empty,
91//! );
92//!
93//! let session_state = SessionStateBuilder::new()
94//! .with_default_features()
95//! .with_physical_optimizer_rule(instrument_rule)
96//! .build();
97//!
98//! let ctx = SessionContext::new_with_state(session_state);
99//!
100//! let results = ctx.sql("SELECT 1").await?.collect().await?;
101//! println!(
102//! "Query Results:\n{}",
103//! pretty_format_batches(results.as_slice())?
104//! );
105//!
106//! Ok(())
107//! }
108//! ```
109//!
110//! A more complete example can be found in the [examples directory](https://github.com/datafusion-contrib/datafusion-tracing/tree/main/examples).
111//!
112
113mod instrument_rule;
114mod instrumented;
115mod instrumented_macros;
116mod metrics;
117mod options;
118mod preview;
119mod preview_utils;
120mod utils;
121
122// Hide implementation details from documentation.
123// This function is only public because it needs to be accessed by the macros,
124// but it's not intended for direct use by consumers of this crate.
125#[doc(hidden)]
126pub use instrument_rule::new_instrument_rule;
127
128pub use options::InstrumentationOptions;
129pub use preview_utils::pretty_format_compact_batch;