datafusion_openlineage/
lib.rs

1//! OpenLineage integration for Apache DataFusion.
2//!
3//! Instrument a [`SessionState`](datafusion::execution::context::SessionState)
4//! with [`OpenLineage::builder`] to emit [OpenLineage](https://openlineage.io)
5//! run events (START / COMPLETE / FAIL) describing each query's input/output
6//! datasets and column-level lineage. Planning-time work (lineage extraction,
7//! context, START) runs in a [`QueryPlanner`](crate::rule::OpenLineageQueryPlanner);
8//! the terminal COMPLETE/FAIL node is installed by a registered
9//! [`ExtensionPlanner`](crate::rule::LineageExtensionPlanner) that lowers a
10//! plan-carried marker — see the [`rule`] module and ADR 0005.
11//!
12//! The event model, the pluggable [`Transport`] seam, and the non-blocking
13//! [`OpenLineageClient`] live in the engine-agnostic
14//! [`openlineage_client`] crate; this crate re-exports them and adds the
15//! DataFusion-specific glue. Orchestration metadata (parent run, job ids, custom
16//! facets) is injected per query via a [`LineageContextProvider`].
17//!
18//! # Quickstart
19//!
20//! ```no_run
21//! use datafusion::execution::SessionStateBuilder;
22//! use datafusion_openlineage::OpenLineage;
23//!
24//! # fn wire() -> Result<(), Box<dyn std::error::Error>> {
25//! let state = SessionStateBuilder::new_with_default_features().build();
26//! // Reads OPENLINEAGE_URL / _API_KEY / _NAMESPACE / parent-run env vars.
27//! let state = OpenLineage::builder().from_env()?.instrument(state);
28//! # let _ = state;
29//! # Ok(())
30//! # }
31//! ```
32#![deny(missing_docs)]
33
34// Event builders. Reachable for integration tests that assert on the emitted
35// event shape, but not part of the advertised API — callers instrument a
36// session rather than building events by hand.
37#[doc(hidden)]
38pub mod builder;
39pub mod column;
40pub mod config;
41pub mod context;
42pub mod exec;
43pub mod extract;
44pub mod rule;
45pub mod session;
46
47// Re-export the engine-agnostic emission surface so the flat
48// `datafusion_openlineage::{RunEvent, Transport, OpenLineageClient, ...}` paths
49// keep working and callers need not depend on `openlineage-client` directly.
50pub use openlineage_client::{
51    ClientError, ConsoleTransport, Dataset, DatasetName, Job, LineageContext, NoopTransport,
52    OpenLineageClient, OpenLineageClientBuilder, OpenLineageConfig, Run, RunEvent, RunEventType,
53    Transport, TransportError, client, event, facets, naming, transport,
54};
55#[cfg(feature = "http")]
56pub use openlineage_client::{CloudClientTransport, cloud};
57
58pub use config::DataFusionConfig;
59pub use context::{LineageContextProvider, StaticContextProvider};
60pub use exec::OpenLineageExec;
61pub use extract::{QueryLineage, extract};
62pub use rule::{
63    LineageExtensionPlanner, LineageHandle, LineageMarker, OpenLineageQueryPlanner, begin_lineage,
64};
65pub use session::{
66    OpenLineage, OpenLineageBuilder, OpenLineageSqlExt, instrument_session_state,
67    instrument_session_state_simple,
68};
datafusion_openlineage/lib.rs

datafusion_openlineage/
lib.rs
