Skip to main content

uqa_execution/maintenance/analyze/
values.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Capture statistics observations and sort their exact Core identities within the caller's allowance.
8
9use std::sync::Arc;
10use uqa_core::{CancellationToken, Value};
11use uqa_sql::SQLError;
12use uqa_storage::{DocumentStore, StorageBackendResult};
13
14use crate::{
15    physical::physical_exec_error, spill::SpillBatchWriter, ExternalSort, PhysicalOperator,
16    PhysicalRow, RowSchema, SpillBuffer, SpillScan,
17};
18
19#[cfg(test)]
20mod tests;
21
22#[derive(Default)]
23pub struct ColumnObservationCounts {
24    pub nulls: u64,
25    pub omitted: u64,
26    pub comparable: u64,
27}
28
29pub struct AnalyzeObservations {
30    pub row_count: u64,
31    pub counts: Vec<ColumnObservationCounts>,
32    pub values: SortedColumnValues,
33}
34
35/// The caller supplies retained snapshots in hierarchy order and the existing value-eligibility policy. Collection finishes before the distribution reader can publish any statistics.
36pub fn collect_values(
37    snapshots: impl IntoIterator<Item = StorageBackendResult<Arc<dyn DocumentStore>>>,
38    columns: &[String],
39    bytes: usize,
40    cancellation: &CancellationToken,
41    accepts: impl Fn(&Value) -> bool,
42) -> Result<AnalyzeObservations, SQLError> {
43    cancellation.check()?;
44    let schema = RowSchema::with_internal_relation_types(
45        uqa_sql::ast::InternalRelationId::allocate(),
46        vec![None; 2],
47    );
48    let mut rows = SpillBuffer::new(bytes / 8);
49    let mut writer = SpillBatchWriter::new(schema.clone()).map_err(physical_exec_error)?;
50    let fields = columns.iter().map(String::as_str).collect::<Vec<_>>();
51    let mut counts = (0..columns.len())
52        .map(|_| ColumnObservationCounts::default())
53        .collect::<Vec<_>>();
54    let mut row_count = 0;
55    for snapshot in snapshots {
56        let snapshot = snapshot.map_err(storage)?;
57        let mut after = None;
58        loop {
59            cancellation.check()?;
60            let ids = snapshot.next_doc_ids(after, 128).map_err(storage)?;
61            if ids.is_empty() {
62                break;
63            }
64            after = ids.last().copied();
65            add(
66                &mut row_count,
67                u64::try_from(ids.len()).map_err(|_| overflow())?,
68            )?;
69            let mut failure = None;
70            let result = snapshot.for_each_fields_multi(&ids, &fields, &mut |_, values| {
71                let result = (|| {
72                    cancellation.check()?;
73                    if values.len() != columns.len() {
74                        return Err(SQLError::Internal(
75                            "ANALYZE projection changed its column count".into(),
76                        ));
77                    }
78                    for (column, value) in values.into_iter().enumerate() {
79                        let count = &mut counts[column];
80                        if matches!(value, Value::Null) {
81                            add(&mut count.nulls, 1)?;
82                        } else if !accepts(&value) {
83                            add(&mut count.omitted, 1)?;
84                        } else {
85                            if matches!(
86                                value,
87                                Value::Int(_) | Value::Float(_) | Value::Str(_) | Value::Bool(_)
88                            ) {
89                                add(&mut count.comparable, 1)?;
90                            }
91                            writer
92                                .push(
93                                    &mut rows,
94                                    PhysicalRow::from_values(vec![
95                                        Value::Int(i64::try_from(column).map_err(|_| overflow())?),
96                                        value,
97                                    ]),
98                                )
99                                .map_err(physical_exec_error)?;
100                        }
101                    }
102                    Ok::<_, SQLError>(())
103                })();
104                if let Err(error) = result {
105                    failure = Some(error);
106                    return false;
107                }
108                true
109            });
110            if let Some(error) = failure {
111                return Err(error);
112            }
113            result.map_err(storage)?;
114        }
115    }
116    writer.finish(&mut rows).map_err(physical_exec_error)?;
117    // Each merge input owns a decoded batch. Share the sort allowance across all input readers and its output instead of granting the entire allowance to every run.
118    let run_bytes = bytes / (crate::external_sort::EXTERNAL_SORT_MERGE_FAN_IN + 2);
119    let mut sorted = ExternalSort::physical_columns(
120        Box::new(SpillScan::new(schema, rows)),
121        vec![0, 1],
122        run_bytes,
123        cancellation,
124    )
125    .map_err(physical_exec_error)?;
126    sorted.open().map_err(physical_exec_error)?;
127    Ok(AnalyzeObservations {
128        row_count,
129        counts,
130        values: SortedColumnValues {
131            sorted,
132            pending: Vec::new().into_iter(),
133            exhausted: false,
134            cancellation: cancellation.clone(),
135        },
136    })
137}
138
139pub struct SortedColumnValues {
140    sorted: ExternalSort<'static>,
141    pending: std::vec::IntoIter<PhysicalRow>,
142    exhausted: bool,
143    cancellation: CancellationToken,
144}
145
146impl Iterator for SortedColumnValues {
147    type Item = Result<(usize, Value), SQLError>;
148
149    fn next(&mut self) -> Option<Self::Item> {
150        if !self.exhausted {
151            if let Err(error) = self.cancellation.check() {
152                self.exhausted = true;
153                return Some(Err(error.into()));
154            }
155        }
156        while !self.exhausted {
157            if let Some(row) = self.pending.next() {
158                return Some((|| {
159                    let values: [Value; 2] = row
160                        .into_physical_values()
161                        .try_into()
162                        .map_err(|_| SQLError::Internal("invalid ANALYZE value width".into()))?;
163                    let [Value::Int(column), value] = values else {
164                        return Err(SQLError::Internal("invalid ANALYZE column identity".into()));
165                    };
166                    Ok((usize::try_from(column).map_err(|_| overflow())?, value))
167                })());
168            }
169            match self.sorted.next() {
170                Ok(Some(batch)) => self.pending = batch.rows.into_iter(),
171                Ok(None) => self.exhausted = true,
172                Err(error) => {
173                    self.exhausted = true;
174                    return Some(Err(physical_exec_error(error)));
175                }
176            }
177        }
178        None
179    }
180}
181
182impl Drop for SortedColumnValues {
183    fn drop(&mut self) {
184        let _ = self.sorted.close();
185    }
186}
187
188fn storage(error: uqa_storage::StorageBackendError) -> SQLError {
189    crate::storage_errors::storage_error("collect ANALYZE values", &error)
190}
191
192fn overflow() -> SQLError {
193    SQLError::Internal("ANALYZE observation count overflow".into())
194}
195
196fn add(target: &mut u64, count: u64) -> Result<(), SQLError> {
197    *target = target.checked_add(count).ok_or_else(overflow)?;
198    Ok(())
199}