Skip to main content

uqa_api/query_builder/
output.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7use super::*;
8
9#[derive(Debug)]
10pub enum QueryBuilderError {
11    Sql(SQLError),
12    Arrow(ArrowError),
13    Parquet(ParquetError),
14    Io(std::io::Error),
15}
16
17impl fmt::Display for QueryBuilderError {
18    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
19        match self {
20            Self::Sql(err) => write!(f, "{err}"),
21            Self::Arrow(err) => write!(f, "{err}"),
22            Self::Parquet(err) => write!(f, "{err}"),
23            Self::Io(err) => write!(f, "{err}"),
24        }
25    }
26}
27
28impl std::error::Error for QueryBuilderError {
29    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
30        match self {
31            Self::Sql(err) => Some(err),
32            Self::Arrow(err) => Some(err),
33            Self::Parquet(err) => Some(err),
34            Self::Io(err) => Some(err),
35        }
36    }
37}
38
39impl From<SQLError> for QueryBuilderError {
40    fn from(value: SQLError) -> Self {
41        Self::Sql(value)
42    }
43}
44
45impl QueryBuilder<'_> {
46    /// Execute the query and convert the result to an Arrow
47    /// [`RecordBatch`]. The batch always includes compatibility
48    /// metadata columns `_doc_id` and `_score` before the requested
49    /// projections.
50    pub fn execute_arrow(&self) -> Result<RecordBatch, QueryBuilderError> {
51        let mut result = self.execute_with_result_metadata()?;
52        // Arrow carries enum values as their labels, as PostgreSQL clients receive them.
53        self.engine.render_enum_labels(&mut result)?;
54        sql_result_to_record_batch(&result).map_err(QueryBuilderError::Arrow)
55    }
56
57    /// Execute the query and write the Arrow result to a Parquet file.
58    pub fn execute_parquet<P: AsRef<Path>>(&self, path: P) -> Result<(), QueryBuilderError> {
59        let batch = self.execute_arrow()?;
60        let file = File::create(path).map_err(QueryBuilderError::Io)?;
61        let mut writer =
62            ArrowWriter::try_new(file, batch.schema(), None).map_err(QueryBuilderError::Parquet)?;
63        writer.write(&batch).map_err(QueryBuilderError::Parquet)?;
64        writer.close().map_err(QueryBuilderError::Parquet)?;
65        Ok(())
66    }
67
68    fn execute_with_result_metadata(&self) -> Result<SQLResult, SQLError> {
69        let mut builder = self.clone();
70        let original = if builder.projections.is_empty() {
71            vec!["*".to_string()]
72        } else {
73            builder.projections
74        };
75        builder.projections = Vec::with_capacity(original.len() + 2);
76        push_projection_once(&mut builder.projections, "_doc_id");
77        push_projection_once(&mut builder.projections, "_score");
78        for projection in original {
79            if projection != "_doc_id" && projection != "_score" {
80                builder.projections.push(projection);
81            }
82        }
83        builder.execute()
84    }
85}
86
87fn push_projection_once(projections: &mut Vec<String>, projection: &str) {
88    if !projections.iter().any(|p| p == projection) {
89        projections.push(projection.to_string());
90    }
91}
92
93pub(super) fn sql_result_to_record_batch(result: &SQLResult) -> Result<RecordBatch, ArrowError> {
94    let fields: Vec<Field> = result
95        .columns
96        .iter()
97        .enumerate()
98        .map(|(column_index, column)| {
99            Field::new(column, infer_arrow_type(column_index, column, result), true)
100        })
101        .collect();
102    let arrays: Vec<ArrayRef> = result
103        .columns
104        .iter()
105        .zip(fields.iter())
106        .enumerate()
107        .map(|(column_index, (column, field))| {
108            build_arrow_array(column_index, column, field.data_type(), result)
109        })
110        .collect::<Result<_, _>>()?;
111    RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays)
112}
113
114pub(super) fn infer_arrow_type(column_index: usize, column: &str, result: &SQLResult) -> DataType {
115    if column == "_doc_id" {
116        return DataType::Int64;
117    }
118    if column == "_score" {
119        return DataType::Float64;
120    }
121
122    let mut ty: Option<DataType> = None;
123    for row_index in 0..result.rows.len() {
124        let Some(value) = result.value_at(row_index, column_index) else {
125            continue;
126        };
127        let next = match value {
128            Value::Null => continue,
129            Value::Bool(_) => DataType::Boolean,
130            Value::Int(_) => DataType::Int64,
131            Value::Float(_) => DataType::Float64,
132            Value::Void
133            | Value::Datum(_)
134            | Value::Enum(_)
135            | Value::Decimal(_)
136            | Value::Str(_)
137            | Value::FixedChar(_)
138            | Value::Json(_)
139            | Value::JsonB(_)
140            | Value::Bytes(_)
141            | Value::BitString(_)
142            | Value::Temporal(_)
143            | Value::Array(_)
144            | Value::LegacyVector(_)
145            | Value::List(_)
146            | Value::Row(_)
147            | Value::Record(_)
148            | Value::Map(_) => DataType::Utf8,
149        };
150        ty = Some(match (ty, next) {
151            (None, dt) => dt,
152            (Some(DataType::Int64), DataType::Float64)
153            | (Some(DataType::Float64), DataType::Int64) => {
154                if column_integers_fit_f64(column_index, result) {
155                    DataType::Float64
156                } else {
157                    DataType::Utf8
158                }
159            }
160            (Some(DataType::Float64), DataType::Float64) => DataType::Float64,
161            (Some(current), dt) if current == dt => current,
162            _ => DataType::Utf8,
163        });
164        if ty == Some(DataType::Utf8) {
165            break;
166        }
167    }
168    ty.unwrap_or(DataType::Utf8)
169}
170
171fn build_arrow_array(
172    column_index: usize,
173    column: &str,
174    data_type: &DataType,
175    result: &SQLResult,
176) -> Result<ArrayRef, ArrowError> {
177    let array: ArrayRef = match data_type {
178        DataType::Boolean => Arc::new(BooleanArray::from(collect_typed_column(
179            column_index,
180            column,
181            result,
182            |value| match value {
183                Value::Bool(value) => Some(*value),
184                _ => None,
185            },
186            "boolean",
187        )?)),
188        DataType::Int64 => Arc::new(Int64Array::from(collect_typed_column(
189            column_index,
190            column,
191            result,
192            |value| match value {
193                Value::Int(value) => Some(*value),
194                _ => None,
195            },
196            "int64",
197        )?)),
198        DataType::Float64 => Arc::new(Float64Array::from(collect_float_column(
199            column_index,
200            column,
201            result,
202        )?)),
203        _ => Arc::new(StringArray::from(
204            (0..result.rows.len())
205                .map(|row_index| {
206                    result
207                        .value_at(row_index, column_index)
208                        .map(value_to_arrow_string)
209                        .transpose()
210                        .map(Option::flatten)
211                })
212                .collect::<Result<Vec<_>, _>>()?,
213        )),
214    };
215    Ok(array)
216}
217
218fn collect_typed_column<T, F>(
219    column_index: usize,
220    column: &str,
221    result: &SQLResult,
222    convert: F,
223    expected: &str,
224) -> Result<Vec<Option<T>>, ArrowError>
225where
226    F: Fn(&Value) -> Option<T>,
227{
228    result
229        .rows
230        .iter()
231        .enumerate()
232        .map(
233            |(row_index, _)| match result.value_at(row_index, column_index) {
234                None | Some(Value::Null) => Ok(None),
235                Some(value) => convert(value).map(Some).ok_or_else(|| {
236                    ArrowError::CastError(format!(
237                        "column `{column}` contains {} where {expected} was inferred",
238                        value_kind(value)
239                    ))
240                }),
241            },
242        )
243        .collect()
244}
245
246fn collect_float_column(
247    column_index: usize,
248    column: &str,
249    result: &SQLResult,
250) -> Result<Vec<Option<f64>>, ArrowError> {
251    result
252        .rows
253        .iter()
254        .enumerate()
255        .map(
256            |(row_index, _)| match result.value_at(row_index, column_index) {
257                None | Some(Value::Null) => Ok(None),
258                Some(Value::Float(value)) => Ok(Some(*value)),
259                Some(Value::Int(value)) => i64_to_f64_exact(*value).map(Some).ok_or_else(|| {
260                    ArrowError::CastError(format!(
261                    "column `{column}` integer {value} cannot be represented exactly as float64"
262                ))
263                }),
264                Some(value) => Err(ArrowError::CastError(format!(
265                    "column `{column}` contains {} where float64 was inferred",
266                    value_kind(value)
267                ))),
268            },
269        )
270        .collect()
271}
272
273fn column_integers_fit_f64(column_index: usize, result: &SQLResult) -> bool {
274    result.rows.iter().enumerate().all(|(row_index, _)| {
275        !matches!(
276            result.value_at(row_index, column_index),
277            Some(Value::Int(value)) if i64_to_f64_exact(*value).is_none()
278        )
279    })
280}
281
282fn i64_to_f64_exact(value: i64) -> Option<f64> {
283    const MAX_SAFE_INTEGER: i64 = 1_i64 << 53;
284    (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER)
285        .contains(&value)
286        .then_some(value as f64)
287}
288
289fn value_kind(value: &Value) -> &'static str {
290    match value {
291        Value::Datum(datum) => uqa_sql::expr::datums::type_name(datum),
292        Value::Null => "null",
293        Value::Void => "void",
294        Value::Bool(_) => "boolean",
295        Value::Int(_) => "integer",
296        Value::Float(_) => "float",
297        Value::Decimal(_) => "decimal",
298        Value::Str(_) | Value::FixedChar(_) => "string",
299        Value::Json(_) => "json",
300        Value::JsonB(_) => "jsonb",
301        Value::Bytes(_) => "bytes",
302        Value::BitString(_) => "bit",
303        Value::Temporal(_) => "temporal",
304        Value::Array(_) => "array",
305        Value::LegacyVector(vector) => vector.kind().type_name(),
306        Value::List(_) => "list",
307        Value::Row(_) => "row",
308        Value::Record(_) => "record",
309        Value::Map(_) => "map",
310        Value::Enum(_) => "enum",
311    }
312}
313
314fn value_to_arrow_string(value: &Value) -> Result<Option<String>, ArrowError> {
315    Ok(match value {
316        Value::Datum(datum) => {
317            return value_to_arrow_string(
318                &uqa_sql::expr::datums::read(datum)
319                    .map_err(|error| ArrowError::CastError(error.to_string()))?,
320            )
321        }
322        Value::Null => None,
323        Value::Void => Some(String::new()),
324        Value::Bool(v) => Some(v.to_string()),
325        Value::Int(v) => Some(v.to_string()),
326        Value::Float(v) => Some(v.to_string()),
327        Value::Decimal(v) => Some(v.to_sql_string()),
328        Value::Str(v) | Value::FixedChar(v) | Value::Json(v) | Value::JsonB(v) => Some(v.clone()),
329        Value::Bytes(v) => Some(format!("{v:?}")),
330        Value::Temporal(v) => Some(v.to_sql_string()),
331        Value::List(v) => Some(format!("{v:?}")),
332        Value::Array(_)
333        | Value::Row(_)
334        | Value::Record(_)
335        | Value::LegacyVector(_)
336        | Value::BitString(_) => {
337            let text = uqa_sql::expr::value_to_string_with_control(
338                value,
339                &uqa_core::memory::ProductionControl::uncontrolled(),
340            )
341            .map_err(|error| {
342                ArrowError::CastError(format!("{}: {error}", error.sqlstate().unwrap_or("XX000")))
343            })?;
344            Some(
345                text.into_uncontrolled()
346                    .expect("ordinary Arrow text output"),
347            )
348        }
349        Value::Map(v) => Some(format!("{v:?}")),
350        Value::Enum(value) => {
351            let error = uqa_sql::expr::catalog_output_required(value);
352            return Err(ArrowError::CastError(format!(
353                "{}: {error}",
354                error.sqlstate().unwrap_or("XX000")
355            )));
356        }
357    })
358}