uqa_execution/maintenance/analyze/
values.rs1use std::sync::Arc;
10use uqa_core::{CancellationToken, Value};
11use uqa_sql::SQLError;
12use uqa_storage::{DocumentStore, StorageBackendResult};
13
14use crate::{
15 physical::physical_exec_error, spill::SpillBatchWriter, ExternalSort, PhysicalOperator,
16 PhysicalRow, RowSchema, SpillBuffer, SpillScan,
17};
18
19#[cfg(test)]
20mod tests;
21
22#[derive(Default)]
23pub struct ColumnObservationCounts {
24 pub nulls: u64,
25 pub omitted: u64,
26 pub comparable: u64,
27}
28
29pub struct AnalyzeObservations {
30 pub row_count: u64,
31 pub counts: Vec<ColumnObservationCounts>,
32 pub values: SortedColumnValues,
33}
34
35pub fn collect_values(
37 snapshots: impl IntoIterator<Item = StorageBackendResult<Arc<dyn DocumentStore>>>,
38 columns: &[String],
39 bytes: usize,
40 cancellation: &CancellationToken,
41 accepts: impl Fn(&Value) -> bool,
42) -> Result<AnalyzeObservations, SQLError> {
43 cancellation.check()?;
44 let schema = RowSchema::with_internal_relation_types(
45 uqa_sql::ast::InternalRelationId::allocate(),
46 vec![None; 2],
47 );
48 let mut rows = SpillBuffer::new(bytes / 8);
49 let mut writer = SpillBatchWriter::new(schema.clone()).map_err(physical_exec_error)?;
50 let fields = columns.iter().map(String::as_str).collect::<Vec<_>>();
51 let mut counts = (0..columns.len())
52 .map(|_| ColumnObservationCounts::default())
53 .collect::<Vec<_>>();
54 let mut row_count = 0;
55 for snapshot in snapshots {
56 let snapshot = snapshot.map_err(storage)?;
57 let mut after = None;
58 loop {
59 cancellation.check()?;
60 let ids = snapshot.next_doc_ids(after, 128).map_err(storage)?;
61 if ids.is_empty() {
62 break;
63 }
64 after = ids.last().copied();
65 add(
66 &mut row_count,
67 u64::try_from(ids.len()).map_err(|_| overflow())?,
68 )?;
69 let mut failure = None;
70 let result = snapshot.for_each_fields_multi(&ids, &fields, &mut |_, values| {
71 let result = (|| {
72 cancellation.check()?;
73 if values.len() != columns.len() {
74 return Err(SQLError::Internal(
75 "ANALYZE projection changed its column count".into(),
76 ));
77 }
78 for (column, value) in values.into_iter().enumerate() {
79 let count = &mut counts[column];
80 if matches!(value, Value::Null) {
81 add(&mut count.nulls, 1)?;
82 } else if !accepts(&value) {
83 add(&mut count.omitted, 1)?;
84 } else {
85 if matches!(
86 value,
87 Value::Int(_) | Value::Float(_) | Value::Str(_) | Value::Bool(_)
88 ) {
89 add(&mut count.comparable, 1)?;
90 }
91 writer
92 .push(
93 &mut rows,
94 PhysicalRow::from_values(vec![
95 Value::Int(i64::try_from(column).map_err(|_| overflow())?),
96 value,
97 ]),
98 )
99 .map_err(physical_exec_error)?;
100 }
101 }
102 Ok::<_, SQLError>(())
103 })();
104 if let Err(error) = result {
105 failure = Some(error);
106 return false;
107 }
108 true
109 });
110 if let Some(error) = failure {
111 return Err(error);
112 }
113 result.map_err(storage)?;
114 }
115 }
116 writer.finish(&mut rows).map_err(physical_exec_error)?;
117 let run_bytes = bytes / (crate::external_sort::EXTERNAL_SORT_MERGE_FAN_IN + 2);
119 let mut sorted = ExternalSort::physical_columns(
120 Box::new(SpillScan::new(schema, rows)),
121 vec![0, 1],
122 run_bytes,
123 cancellation,
124 )
125 .map_err(physical_exec_error)?;
126 sorted.open().map_err(physical_exec_error)?;
127 Ok(AnalyzeObservations {
128 row_count,
129 counts,
130 values: SortedColumnValues {
131 sorted,
132 pending: Vec::new().into_iter(),
133 exhausted: false,
134 cancellation: cancellation.clone(),
135 },
136 })
137}
138
139pub struct SortedColumnValues {
140 sorted: ExternalSort<'static>,
141 pending: std::vec::IntoIter<PhysicalRow>,
142 exhausted: bool,
143 cancellation: CancellationToken,
144}
145
146impl Iterator for SortedColumnValues {
147 type Item = Result<(usize, Value), SQLError>;
148
149 fn next(&mut self) -> Option<Self::Item> {
150 if !self.exhausted {
151 if let Err(error) = self.cancellation.check() {
152 self.exhausted = true;
153 return Some(Err(error.into()));
154 }
155 }
156 while !self.exhausted {
157 if let Some(row) = self.pending.next() {
158 return Some((|| {
159 let values: [Value; 2] = row
160 .into_physical_values()
161 .try_into()
162 .map_err(|_| SQLError::Internal("invalid ANALYZE value width".into()))?;
163 let [Value::Int(column), value] = values else {
164 return Err(SQLError::Internal("invalid ANALYZE column identity".into()));
165 };
166 Ok((usize::try_from(column).map_err(|_| overflow())?, value))
167 })());
168 }
169 match self.sorted.next() {
170 Ok(Some(batch)) => self.pending = batch.rows.into_iter(),
171 Ok(None) => self.exhausted = true,
172 Err(error) => {
173 self.exhausted = true;
174 return Some(Err(physical_exec_error(error)));
175 }
176 }
177 }
178 None
179 }
180}
181
182impl Drop for SortedColumnValues {
183 fn drop(&mut self) {
184 let _ = self.sorted.close();
185 }
186}
187
188fn storage(error: uqa_storage::StorageBackendError) -> SQLError {
189 crate::storage_errors::storage_error("collect ANALYZE values", &error)
190}
191
192fn overflow() -> SQLError {
193 SQLError::Internal("ANALYZE observation count overflow".into())
194}
195
196fn add(target: &mut u64, count: u64) -> Result<(), SQLError> {
197 *target = target.checked_add(count).ok_or_else(overflow)?;
198 Ok(())
199}