uni_plugin_pyo3/
lib.rs

1//! PyO3 live-callable plugin loader for uni-db.
2//!
3//! This crate bridges Python callables held in the host process to the
4//! [`uni_plugin`] trait surfaces. Plugins are **session-scoped by
5//! default** (`Scope::Session`) and run at host process privilege
6//! (**no sandbox**). Capabilities are declared metadata and enforced
7//! at the [`PluginRegistrar`](uni_plugin::PluginRegistrar) gate; there
8//! is no structural sandbox layer for PyO3 because the callable is a
9//! live Python object in the host process.
10//!
11//! # Crate layout
12//!
13//! - `error` — `PyPluginError`, the structured error type. `From<PyErr>`
14//!   captures Python tracebacks under the GIL.
15//! - `arrow_bridge` — Arrow ↔ PyArrow zero-copy via the
16//!   [Arrow PyCapsule Interface]. No `pyo3-arrow` dependency.
17//! - Manifest / loader / adapters land in later M8 sub-milestones.
18//!
19//! [Arrow PyCapsule Interface]: https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html
20//!
21//! # Loader execution model
22//!
23//! Two GIL strategies map onto the proposal's two modes:
24//!
25//! - **Vectorized** (`vectorized=True`): one `Python::with_gil` per
26//!   RecordBatch. Each input column is marshaled to a pyarrow Array via
27//!   the PyCapsule protocol (zero-copy); the user fn runs once per batch
28//!   and returns a pyarrow Array; the result is marshaled back to Arrow.
29//!   Recommended ceiling: ~5M+ rows/sec on trivial Float64 fns over
30//!   8192-row batches.
31//!
32//! - **Row-by-row** (`vectorized=False`): one `Python::with_gil` *per
33//!   batch* still (we hold the GIL across the rows in a batch — design
34//!   decision #6 in `plans/magical-rolling-pinwheel.md`); inside the
35//!   closure the host iterates rows and calls the Python fn once per
36//!   row with native PyObject args. Approximate ceiling: ~100k
37//!   rows/sec.
38//!
39//! Both modes serialize on the GIL, so a multi-partition DataFusion
40//! scan with a PyO3 UDF collapses to single-core throughput. This is
41//! the **dominant operational concern** with PyO3 UDFs and is
42//! documented in the proposal at §5.4.1. Mitigations
43//! (sub-interpreter parallelism, free-threading) are deferred to
44//! follow-up milestones.
45
46// Rust guideline compliant
47#![warn(missing_docs)]
48#![warn(rust_2018_idioms)]
49#![warn(missing_debug_implementations)]
50
51pub mod error;
52
53#[cfg(feature = "pyo3")]
54pub mod adapter_aggregate;
55#[cfg(feature = "pyo3")]
56pub mod adapter_procedure;
57#[cfg(feature = "pyo3")]
58pub mod adapter_scalar;
59#[cfg(feature = "pyo3")]
60pub(crate) mod adapter_scalar_helpers;
61#[cfg(feature = "pyo3")]
62pub mod arrow_bridge;
63#[cfg(feature = "pyo3")]
64pub mod loader;
65#[cfg(feature = "pyo3")]
66pub mod manifest;
67#[cfg(feature = "pyo3")]
68pub mod plugin_handle;
69#[cfg(feature = "pyo3")]
70pub mod runtime;
71
72#[doc(inline)]
73pub use crate::error::PyPluginError;
74
75#[cfg(feature = "pyo3")]
76#[doc(inline)]
77pub use crate::adapter_aggregate::{PyAccumulator, PyAggregateFn, build_py_agg_signature};
78#[cfg(feature = "pyo3")]
79#[doc(inline)]
80pub use crate::adapter_procedure::PyProcedure;
81#[cfg(feature = "pyo3")]
82#[doc(inline)]
83pub use crate::adapter_scalar::PyScalarFn;
84#[cfg(feature = "pyo3")]
85#[doc(inline)]
86pub use crate::loader::{
87    LoadOutcome, PyDecoratorSink, PyDecoratorTrampoline, PyPluginLoader as PythonPluginLoader,
88    make_aggregate_trampoline, make_procedure_trampoline, make_scalar_trampoline,
89};
90#[cfg(feature = "pyo3")]
91#[doc(inline)]
92pub use crate::manifest::{
93    ManifestBuilder, PyAggregateEntry, PyManifest, PyProcedureEntry, PyScalarEntry,
94};
95#[cfg(feature = "pyo3")]
96#[doc(inline)]
97pub use crate::plugin_handle::PyPluginHandle;
98#[cfg(feature = "pyo3")]
99#[doc(inline)]
100pub use crate::runtime::PyPluginRuntime;
uni_plugin_pyo3/lib.rs

uni_plugin_pyo3/
lib.rs
