Skip to main content

uqa_api/query_builder/
output.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7use super::*;
8
9#[derive(Debug)]
10pub enum QueryBuilderError {
11    Sql(SQLError),
12    Arrow(ArrowError),
13    Parquet(ParquetError),
14    Io(std::io::Error),
15}
16
17impl fmt::Display for QueryBuilderError {
18    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
19        match self {
20            Self::Sql(err) => write!(f, "{err}"),
21            Self::Arrow(err) => write!(f, "{err}"),
22            Self::Parquet(err) => write!(f, "{err}"),
23            Self::Io(err) => write!(f, "{err}"),
24        }
25    }
26}
27
28impl std::error::Error for QueryBuilderError {
29    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
30        match self {
31            Self::Sql(err) => Some(err),
32            Self::Arrow(err) => Some(err),
33            Self::Parquet(err) => Some(err),
34            Self::Io(err) => Some(err),
35        }
36    }
37}
38
39impl From<SQLError> for QueryBuilderError {
40    fn from(value: SQLError) -> Self {
41        Self::Sql(value)
42    }
43}
44
45impl QueryBuilder<'_> {
46    /// Execute the query and convert the result to an Arrow
47    /// [`RecordBatch`]. The batch always includes compatibility
48    /// metadata columns `_doc_id` and `_score` before the requested
49    /// projections.
50    pub fn execute_arrow(&self) -> Result<RecordBatch, QueryBuilderError> {
51        let mut result = self.execute_with_result_metadata()?;
52        // Arrow carries enum values as their labels, as PostgreSQL clients receive them.
53        self.engine.render_enum_labels(&mut result)?;
54        sql_result_to_record_batch(&result).map_err(QueryBuilderError::Arrow)
55    }
56
57    /// Execute the query and write the Arrow result to a Parquet file.
58    pub fn execute_parquet<P: AsRef<Path>>(&self, path: P) -> Result<(), QueryBuilderError> {
59        let batch = self.execute_arrow()?;
60        let file = File::create(path).map_err(QueryBuilderError::Io)?;
61        let mut writer =
62            ArrowWriter::try_new(file, batch.schema(), None).map_err(QueryBuilderError::Parquet)?;
63        writer.write(&batch).map_err(QueryBuilderError::Parquet)?;
64        writer.close().map_err(QueryBuilderError::Parquet)?;
65        Ok(())
66    }
67
68    fn execute_with_result_metadata(&self) -> Result<SQLResult, SQLError> {
69        let mut builder = self.clone();
70        let original = if builder.projections.is_empty() {
71            vec!["*".to_string()]
72        } else {
73            builder.projections
74        };
75        builder.projections = Vec::with_capacity(original.len() + 2);
76        push_projection_once(&mut builder.projections, "_doc_id");
77        push_projection_once(&mut builder.projections, "_score");
78        for projection in original {
79            if projection != "_doc_id" && projection != "_score" {
80                builder.projections.push(projection);
81            }
82        }
83        builder.execute()
84    }
85}
86
87fn push_projection_once(projections: &mut Vec<String>, projection: &str) {
88    if !projections.iter().any(|p| p == projection) {
89        projections.push(projection.to_string());
90    }
91}
92
93pub(super) fn sql_result_to_record_batch(result: &SQLResult) -> Result<RecordBatch, ArrowError> {
94    let fields: Vec<Field> = result
95        .columns
96        .iter()
97        .enumerate()
98        .map(|(column_index, column)| {
99            Field::new(column, infer_arrow_type(column_index, column, result), true)
100        })
101        .collect();
102    let arrays: Vec<ArrayRef> = result
103        .columns
104        .iter()
105        .zip(fields.iter())
106        .enumerate()
107        .map(|(column_index, (column, field))| {
108            build_arrow_array(column_index, column, field.data_type(), result)
109        })
110        .collect::<Result<_, _>>()?;
111    RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays)
112}
113
114pub(super) fn infer_arrow_type(column_index: usize, column: &str, result: &SQLResult) -> DataType {
115    if column == "_doc_id" {
116        return DataType::Int64;
117    }
118    if column == "_score" {
119        return DataType::Float64;
120    }
121
122    let mut ty: Option<DataType> = None;
123    for row_index in 0..result.rows.len() {
124        let Some(value) = result.value_at(row_index, column_index) else {
125            continue;
126        };
127        let next = match value {
128            Value::Null => continue,
129            Value::Bool(_) => DataType::Boolean,
130            Value::Int(_) => DataType::Int64,
131            Value::Float(_) => DataType::Float64,
132            Value::Void
133            | Value::Datum(_)
134            | Value::Enum(_)
135            | Value::Decimal(_)
136            | Value::Str(_)
137            | Value::FixedChar(_)
138            | Value::Json(_)
139            | Value::JsonB(_)
140            | Value::Bytes(_)
141            | Value::Temporal(_)
142            | Value::Array(_)
143            | Value::LegacyVector(_)
144            | Value::List(_)
145            | Value::Row(_)
146            | Value::Record(_)
147            | Value::Map(_) => DataType::Utf8,
148        };
149        ty = Some(match (ty, next) {
150            (None, dt) => dt,
151            (Some(DataType::Int64), DataType::Float64)
152            | (Some(DataType::Float64), DataType::Int64) => {
153                if column_integers_fit_f64(column_index, result) {
154                    DataType::Float64
155                } else {
156                    DataType::Utf8
157                }
158            }
159            (Some(DataType::Float64), DataType::Float64) => DataType::Float64,
160            (Some(current), dt) if current == dt => current,
161            _ => DataType::Utf8,
162        });
163        if ty == Some(DataType::Utf8) {
164            break;
165        }
166    }
167    ty.unwrap_or(DataType::Utf8)
168}
169
170fn build_arrow_array(
171    column_index: usize,
172    column: &str,
173    data_type: &DataType,
174    result: &SQLResult,
175) -> Result<ArrayRef, ArrowError> {
176    let array: ArrayRef = match data_type {
177        DataType::Boolean => Arc::new(BooleanArray::from(collect_typed_column(
178            column_index,
179            column,
180            result,
181            |value| match value {
182                Value::Bool(value) => Some(*value),
183                _ => None,
184            },
185            "boolean",
186        )?)),
187        DataType::Int64 => Arc::new(Int64Array::from(collect_typed_column(
188            column_index,
189            column,
190            result,
191            |value| match value {
192                Value::Int(value) => Some(*value),
193                _ => None,
194            },
195            "int64",
196        )?)),
197        DataType::Float64 => Arc::new(Float64Array::from(collect_float_column(
198            column_index,
199            column,
200            result,
201        )?)),
202        _ => Arc::new(StringArray::from(
203            (0..result.rows.len())
204                .map(|row_index| {
205                    result
206                        .value_at(row_index, column_index)
207                        .map(value_to_arrow_string)
208                        .transpose()
209                        .map(Option::flatten)
210                })
211                .collect::<Result<Vec<_>, _>>()?,
212        )),
213    };
214    Ok(array)
215}
216
217fn collect_typed_column<T, F>(
218    column_index: usize,
219    column: &str,
220    result: &SQLResult,
221    convert: F,
222    expected: &str,
223) -> Result<Vec<Option<T>>, ArrowError>
224where
225    F: Fn(&Value) -> Option<T>,
226{
227    result
228        .rows
229        .iter()
230        .enumerate()
231        .map(
232            |(row_index, _)| match result.value_at(row_index, column_index) {
233                None | Some(Value::Null) => Ok(None),
234                Some(value) => convert(value).map(Some).ok_or_else(|| {
235                    ArrowError::CastError(format!(
236                        "column `{column}` contains {} where {expected} was inferred",
237                        value_kind(value)
238                    ))
239                }),
240            },
241        )
242        .collect()
243}
244
245fn collect_float_column(
246    column_index: usize,
247    column: &str,
248    result: &SQLResult,
249) -> Result<Vec<Option<f64>>, ArrowError> {
250    result
251        .rows
252        .iter()
253        .enumerate()
254        .map(
255            |(row_index, _)| match result.value_at(row_index, column_index) {
256                None | Some(Value::Null) => Ok(None),
257                Some(Value::Float(value)) => Ok(Some(*value)),
258                Some(Value::Int(value)) => i64_to_f64_exact(*value).map(Some).ok_or_else(|| {
259                    ArrowError::CastError(format!(
260                    "column `{column}` integer {value} cannot be represented exactly as float64"
261                ))
262                }),
263                Some(value) => Err(ArrowError::CastError(format!(
264                    "column `{column}` contains {} where float64 was inferred",
265                    value_kind(value)
266                ))),
267            },
268        )
269        .collect()
270}
271
272fn column_integers_fit_f64(column_index: usize, result: &SQLResult) -> bool {
273    result.rows.iter().enumerate().all(|(row_index, _)| {
274        !matches!(
275            result.value_at(row_index, column_index),
276            Some(Value::Int(value)) if i64_to_f64_exact(*value).is_none()
277        )
278    })
279}
280
281fn i64_to_f64_exact(value: i64) -> Option<f64> {
282    const MAX_SAFE_INTEGER: i64 = 1_i64 << 53;
283    (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER)
284        .contains(&value)
285        .then_some(value as f64)
286}
287
288fn value_kind(value: &Value) -> &'static str {
289    match value {
290        Value::Datum(datum) => uqa_sql::expr::datums::type_name(datum),
291        Value::Null => "null",
292        Value::Void => "void",
293        Value::Bool(_) => "boolean",
294        Value::Int(_) => "integer",
295        Value::Float(_) => "float",
296        Value::Decimal(_) => "decimal",
297        Value::Str(_) | Value::FixedChar(_) => "string",
298        Value::Json(_) => "json",
299        Value::JsonB(_) => "jsonb",
300        Value::Bytes(_) => "bytes",
301        Value::Temporal(_) => "temporal",
302        Value::Array(_) => "array",
303        Value::LegacyVector(vector) => vector.kind().type_name(),
304        Value::List(_) => "list",
305        Value::Row(_) => "row",
306        Value::Record(_) => "record",
307        Value::Map(_) => "map",
308        Value::Enum(_) => "enum",
309    }
310}
311
312fn value_to_arrow_string(value: &Value) -> Result<Option<String>, ArrowError> {
313    Ok(match value {
314        Value::Datum(datum) => {
315            return value_to_arrow_string(
316                &uqa_sql::expr::datums::read(datum)
317                    .map_err(|error| ArrowError::CastError(error.to_string()))?,
318            )
319        }
320        Value::Null => None,
321        Value::Void => Some(String::new()),
322        Value::Bool(v) => Some(v.to_string()),
323        Value::Int(v) => Some(v.to_string()),
324        Value::Float(v) => Some(v.to_string()),
325        Value::Decimal(v) => Some(v.to_sql_string()),
326        Value::Str(v) | Value::FixedChar(v) | Value::Json(v) | Value::JsonB(v) => Some(v.clone()),
327        Value::Bytes(v) => Some(format!("{v:?}")),
328        Value::Temporal(v) => Some(v.to_sql_string()),
329        Value::List(v) => Some(format!("{v:?}")),
330        Value::Array(_) | Value::Row(_) | Value::Record(_) | Value::LegacyVector(_) => {
331            let text = uqa_sql::expr::value_to_string_with_control(
332                value,
333                &uqa_core::memory::ProductionControl::uncontrolled(),
334            )
335            .map_err(|error| {
336                ArrowError::CastError(format!("{}: {error}", error.sqlstate().unwrap_or("XX000")))
337            })?;
338            Some(
339                text.into_uncontrolled()
340                    .expect("ordinary Arrow text output"),
341            )
342        }
343        Value::Map(v) => Some(format!("{v:?}")),
344        Value::Enum(value) => {
345            let error = uqa_sql::expr::catalog_output_required(value);
346            return Err(ArrowError::CastError(format!(
347                "{}: {error}",
348                error.sqlstate().unwrap_or("XX000")
349            )));
350        }
351    })
352}