Skip to main content

uqa_api/
arrow_export.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Descriptor-directed Arrow output, independent of query-builder compatibility columns.
8
9use arrow_array::{ArrayRef, RecordBatch, RecordBatchOptions};
10use arrow_schema::{DataType, Field, Schema};
11use std::{collections::HashMap, sync::Arc};
12use uqa_core::{
13    memory::{Produced, ProductionControl},
14    Value,
15};
16use uqa_sql::{catalog::type_metadata, expr::EngineHook, ColumnType, SQLError, SQLResult};
17
18mod columns;
19
20/// Output policy is explicit; unsupported lossless mappings never become text implicitly.
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
22pub enum ArrowExportPolicy {
23    Lossless,
24    SQLText,
25}
26
27/// Borrow a positional result or retained batch without copying its values or resolving names.
28pub trait ArrowRows {
29    fn row_count(&self) -> usize;
30    fn column_count(&self) -> usize;
31    fn column_name(&self, column: usize) -> &str;
32    fn column_type(&self, column: usize) -> Option<&ColumnType>;
33    fn value_at(&self, row: usize, column: usize) -> Option<&Value>;
34}
35
36impl ArrowRows for SQLResult {
37    fn row_count(&self) -> usize {
38        self.rows.len()
39    }
40
41    fn column_count(&self) -> usize {
42        self.columns.len()
43    }
44
45    fn column_name(&self, column: usize) -> &str {
46        &self.columns[column]
47    }
48
49    fn column_type(&self, column: usize) -> Option<&ColumnType> {
50        self.column_types.get(column).and_then(Option::as_ref)
51    }
52
53    fn value_at(&self, row: usize, column: usize) -> Option<&Value> {
54        self.value_at(row, column)
55    }
56}
57
58fn unsupported(message: &str) -> SQLError {
59    SQLError::Unsupported(message.into())
60}
61
62fn overflow() -> SQLError {
63    SQLError::Routine {
64        sqlstate: "54000".into(),
65        message: "Arrow export exceeds addressable memory".into(),
66    }
67}
68
69fn arrow_error(error: &arrow_schema::ArrowError) -> SQLError {
70    SQLError::Internal(format!("construct Arrow result: {error}"))
71}
72
73fn data_type(ty: Option<&ColumnType>, policy: ArrowExportPolicy) -> Result<DataType, SQLError> {
74    if policy == ArrowExportPolicy::SQLText {
75        return Ok(DataType::LargeUtf8);
76    }
77    Ok(match ty {
78        Some(ColumnType::Boolean) => DataType::Boolean,
79        Some(ColumnType::SmallInteger) => DataType::Int16,
80        Some(ColumnType::Integer) => DataType::Int32,
81        Some(ColumnType::BigInteger) => DataType::Int64,
82        Some(ColumnType::Real) => DataType::Float32,
83        Some(ColumnType::DoublePrecision) => DataType::Float64,
84        Some(
85            ColumnType::Text
86            | ColumnType::Bpchar
87            | ColumnType::Character(_)
88            | ColumnType::Varchar(_)
89            | ColumnType::Json
90            | ColumnType::JsonB,
91        ) => DataType::LargeUtf8,
92        Some(ColumnType::Bytea) => DataType::LargeBinary,
93        Some(ColumnType::Uuid) => DataType::FixedSizeBinary(16),
94        Some(ColumnType::Numeric {
95            precision: Some(precision),
96            scale,
97        }) => {
98            let precision = u8::try_from(*precision)
99                .map_err(|_| unsupported("NUMERIC precision has no lossless Arrow mapping"))?;
100            let scale = i8::try_from(scale.unwrap_or(0))
101                .map_err(|_| unsupported("NUMERIC scale has no lossless Arrow mapping"))?;
102            arrow_array::types::validate_decimal_precision_and_scale::<
103                arrow_array::types::Decimal128Type,
104            >(precision, scale)
105            .map_err(|_| unsupported("NUMERIC modifier has no lossless Arrow decimal mapping"))?;
106            DataType::Decimal128(precision, scale)
107        }
108        _ => {
109            return Err(unsupported(
110                "SQL type has no qualified lossless Arrow mapping",
111            ))
112        }
113    })
114}
115
116fn field(name: &str, ty: Option<&ColumnType>, output: DataType) -> Field {
117    let mut metadata = HashMap::with_capacity(4);
118    metadata.insert(
119        "uqa.sql.type_oid".into(),
120        ty.map_or(0, type_metadata::pg_type_oid).to_string(),
121    );
122    metadata.insert(
123        "uqa.sql.type_name".into(),
124        ty.map_or("unknown", type_metadata::column_type_name).into(),
125    );
126    metadata.insert(
127        "uqa.sql.type_modifier".into(),
128        ty.map_or(-1, type_metadata::pg_type_modifier).to_string(),
129    );
130    let schema = match ty {
131        Some(ColumnType::Domain { schema, .. }) => Some(schema.as_str()),
132        Some(ColumnType::Enum(reference)) => Some(reference.schema.as_str()),
133        Some(ColumnType::Composite(reference)) => Some(reference.schema.as_str()),
134        Some(ColumnType::DeclaredRange(reference) | ColumnType::DeclaredMultirange(reference)) => {
135            Some(reference.schema.as_str())
136        }
137        _ => None,
138    };
139    if let Some(schema) = schema {
140        metadata.insert("uqa.sql.type_schema".into(), schema.into());
141    }
142    Field::new(name, output, true).with_metadata(metadata)
143}
144
145/// Convert under the caller's allowance and both cancellation tokens. The returned
146/// lease owns the buffers independently of the original result and session.
147pub fn export_arrow(
148    source: &dyn ArrowRows,
149    policy: ArrowExportPolicy,
150    engine: Option<&dyn EngineHook>,
151    control: &ProductionControl<'_>,
152) -> Result<Produced<RecordBatch>, SQLError> {
153    control.check()?;
154    // Arrow owns its allocation strategy. Admit the complete fixed-width buffers,
155    // validity/offset padding and metadata before invoking its constructors. Text
156    // and binary payload capacity is separately admitted after controlled formatting.
157    let row_bytes = source.row_count().checked_mul(24).ok_or_else(overflow)?;
158    let mut fixed = 1024usize;
159    for column in 0..source.column_count() {
160        control.check()?;
161        data_type(source.column_type(column), policy)?;
162        let ty = source.column_type(column);
163        let type_name = ty.map_or("unknown", type_metadata::column_type_name);
164        // A field holds four metadata entries, an Arc, strings, an array owner and
165        // at most three buffers, each rounded to Arrow's 64-byte alignment.
166        let metadata = source
167            .column_name(column)
168            .len()
169            .checked_add(type_name.len())
170            .and_then(|n| n.checked_mul(4))
171            .and_then(|n| n.checked_add(4096))
172            .ok_or_else(overflow)?;
173        fixed = fixed
174            .checked_add(metadata)
175            .and_then(|n| n.checked_add(row_bytes))
176            .ok_or_else(overflow)?;
177    }
178    let mut memory = control.reserve(fixed)?;
179    let mut fields = Vec::with_capacity(source.column_count());
180    let mut arrays: Vec<ArrayRef> = Vec::with_capacity(source.column_count());
181    for column in 0..source.column_count() {
182        control.check()?;
183        let ty = source.column_type(column);
184        let output = data_type(ty, policy)?;
185        let array = columns::build(source, column, &output, policy, engine, control)?;
186        let (array, payload) = array.into_parts();
187        memory = control.combine(memory, payload);
188        arrays.push(array);
189        fields.push(field(source.column_name(column), ty, output));
190    }
191    let batch = RecordBatch::try_new_with_options(
192        Arc::new(Schema::new(fields)),
193        arrays,
194        &RecordBatchOptions::new().with_row_count(Some(source.row_count())),
195    )
196    .map_err(|error| arrow_error(&error))?;
197    Ok(control.finish(batch, memory)?)
198}