Skip to main content

uqa_execution/
copy.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Execute COPY byte streams through the ordinary INSERT and SELECT statement paths.
8use crate::{query::output::QueryRows, statement::plan_executor::StatementOutput};
9use std::io::{Read, Write};
10use uqa_core::{
11    memory::{MemoryBudget, ProductionControl},
12    CancellationToken, Value,
13};
14use uqa_sql::{
15    ast::{Expr, InsertStmt, ReturningAliases, Statement},
16    catalog::security::table::TableAclPrivilege,
17    copy::stream::CopyCatalog,
18    copy::{
19        compile_copy, decode_copy_input, write_copy_result, CopyDirection, CopyEncoder, CopyTarget,
20    },
21    expr::EngineHook,
22    SQLError, SQLParam, SQLResult,
23};
24pub trait CopyPrivileges {
25    fn ensure_any_column(&self, table: &str, privilege: TableAclPrivilege) -> Result<(), SQLError>;
26    fn ensure_column(
27        &self,
28        table: &str,
29        column: &str,
30        privilege: TableAclPrivilege,
31    ) -> Result<(), SQLError>;
32}
33pub trait CopyStatements {
34    fn execute_statement(
35        &self,
36        statement: Statement,
37        params: &[SQLParam],
38    ) -> Result<SQLResult, SQLError>;
39    fn execute_text_with_output(
40        &self,
41        text: &str,
42        params: &[SQLParam],
43        consume: &mut dyn FnMut(StatementOutput) -> Result<(), SQLError>,
44    ) -> Result<(), SQLError>;
45}
46pub struct CopyExecutionContext<'a> {
47    pub catalog: &'a dyn CopyCatalog,
48    pub privileges: &'a dyn CopyPrivileges,
49    pub statements: &'a dyn CopyStatements,
50    pub output: &'a dyn EngineHook,
51    pub memory: &'a dyn crate::query::runtime::QueryMemorySettings,
52    pub cancellation: &'a CancellationToken,
53}
54pub fn copy_from(
55    context: &CopyExecutionContext<'_>,
56    statement: &str,
57    input: &mut impl Read,
58) -> Result<u64, SQLError> {
59    let copy = compile_copy(statement)?;
60    uqa_sql::copy::stream::validate_stream(&copy, CopyDirection::From)?;
61    let CopyTarget::Relation {
62        name: relation,
63        qualifier,
64        columns: requested_columns,
65    } = &copy.target
66    else {
67        return Err(SQLError::Routine {
68            sqlstate: "42601".into(),
69            message: "COPY FROM requires a relation target".into(),
70        });
71    };
72    let (canonical, columns) = uqa_sql::copy::stream::relation_columns(
73        context.catalog,
74        relation,
75        qualifier,
76        requested_columns,
77        false,
78    )?;
79    if columns.is_empty() {
80        context
81            .privileges
82            .ensure_any_column(&canonical, TableAclPrivilege::Insert)?;
83    } else {
84        for column in &columns {
85            context
86                .privileges
87                .ensure_column(&canonical, column, TableAclPrivilege::Insert)?;
88        }
89    }
90    let mut bytes = Vec::new();
91    input
92        .read_to_end(&mut bytes)
93        .map_err(|error| copy_io_error("read COPY FROM stream", error))?;
94    let rows = decode_copy_input(&bytes, &copy.options, &columns)?;
95    if rows.is_empty() {
96        return Ok(0);
97    }
98    if columns.is_empty() {
99        return Err(SQLError::Unsupported(
100            "COPY FROM for a zero-column relation is not implemented".into(),
101        ));
102    }
103    let mut params = Vec::with_capacity(rows.len().saturating_mul(columns.len()));
104    let mut parameter = 1usize;
105    let mut insert_rows = Vec::with_capacity(rows.len());
106    for row in rows {
107        let mut values = Vec::with_capacity(row.len());
108        for field in row {
109            values.push(Expr::Param(parameter));
110            parameter = parameter.checked_add(1).ok_or_else(|| SQLError::Routine {
111                sqlstate: "54000".into(),
112                message: "COPY input has too many fields".into(),
113            })?;
114            params.push(SQLParam::Scalar(match field {
115                Some(value) => Value::Str(value),
116                None => Value::Null,
117            }));
118        }
119        insert_rows.push(values);
120    }
121    let result = context.statements.execute_statement(
122        Statement::Insert(InsertStmt {
123            table: relation.clone(),
124            target_relation_bound: false,
125            target_qualifier: qualifier.clone(),
126            target_alias: None,
127            include_descendants: true,
128            columns: columns.into_iter().map(Into::into).collect(),
129            // COPY FROM writes the values its input supplies, also to a GENERATED ALWAYS identity column.
130            overriding: Some(uqa_sql::ast::OverridingKind::SystemValue),
131            with: Vec::new(),
132            rows: insert_rows,
133            select_source: None,
134            on_conflict: None,
135            returning: Vec::new(),
136            returning_aliases: ReturningAliases::default(),
137        }),
138        &params,
139    )?;
140    Ok(result.affected_rows)
141}
142
143pub fn copy_to(
144    context: &CopyExecutionContext<'_>,
145    statement: &str,
146    output: &mut impl Write,
147) -> Result<u64, SQLError> {
148    let copy = compile_copy(statement)?;
149    uqa_sql::copy::stream::validate_stream(&copy, CopyDirection::To)?;
150    let query = match &copy.target {
151        CopyTarget::Relation {
152            name: relation,
153            qualifier,
154            columns: requested_columns,
155        } => {
156            let (canonical, columns) = uqa_sql::copy::stream::relation_columns(
157                context.catalog,
158                relation,
159                qualifier,
160                requested_columns,
161                true,
162            )?;
163            if columns.is_empty() {
164                context
165                    .privileges
166                    .ensure_any_column(&canonical, TableAclPrivilege::Select)?;
167            } else {
168                for column in &columns {
169                    context.privileges.ensure_column(
170                        &canonical,
171                        column,
172                        TableAclPrivilege::Select,
173                    )?;
174                }
175            }
176            let projection = columns
177                .iter()
178                .map(|column| uqa_sql::expr::quote_ident(column))
179                .collect::<Vec<_>>()
180                .join(", ");
181            format!("SELECT {projection} FROM ONLY {relation}")
182        }
183        CopyTarget::Query(query) => query.clone(),
184    };
185    let mut row_count = 0;
186    context
187        .statements
188        .execute_text_with_output(&query, &[], &mut |result| {
189            let memory = MemoryBudget::new(context.memory.work_mem_bytes()?);
190            let control =
191                ProductionControl::new(&memory, context.cancellation, context.cancellation);
192            row_count = write_statement_output(
193                result,
194                &copy.options,
195                output,
196                Some(context.output),
197                &control,
198            )?;
199            Ok(())
200        })?;
201    Ok(row_count)
202}
203
204pub(crate) fn write_statement_output(
205    result: StatementOutput,
206    options: &uqa_sql::copy::CopyOptions,
207    output: &mut impl Write,
208    engine: Option<&dyn EngineHook>,
209    control: &ProductionControl<'_>,
210) -> Result<u64, SQLError> {
211    match result {
212        StatementOutput::Command(result) => {
213            let count = copy_row_count(result.rows.len())?;
214            write_copy_result(&result, options, output, engine, control)?;
215            Ok(count)
216        }
217        StatementOutput::Mutation(result) => {
218            let count = copy_row_count(result.row_count())?;
219            let schema = result.schema.clone();
220            write_copy_batches(
221                schema.columns(),
222                schema.column_types(),
223                result.into_batches(),
224                options,
225                output,
226                engine,
227                control,
228            )?;
229            Ok(count)
230        }
231        StatementOutput::Query(result) => {
232            let QueryRows::SharedSpill(rows) = result.rows else {
233                return Err(SQLError::Internal(
234                    "COPY query returned unbounded rows".into(),
235                ));
236            };
237            let count = copy_row_count(rows.rows())?;
238            let rows = rows
239                .into_reader()
240                .map_err(crate::physical::physical_exec_error)?;
241            write_copy_batches(
242                &result.columns,
243                &result.column_types,
244                rows.map(|batch| batch.map_err(crate::physical::physical_exec_error)),
245                options,
246                output,
247                engine,
248                control,
249            )?;
250            Ok(count)
251        }
252    }
253}
254
255fn write_copy_batches(
256    columns: &[String],
257    column_types: &[Option<uqa_sql::ast::ColumnType>],
258    mut batches: impl Iterator<Item = Result<crate::Batch, SQLError>>,
259    options: &uqa_sql::copy::CopyOptions,
260    output: &mut impl Write,
261    engine: Option<&dyn EngineHook>,
262    control: &ProductionControl<'_>,
263) -> Result<(), SQLError> {
264    let mut encoder = CopyEncoder::new(options, output, engine, *control)?;
265    encoder.write_header(columns)?;
266    loop {
267        control.check()?;
268        let Some(batch) = batches.next() else { break };
269        let batch = batch?;
270        for row in batch.rows {
271            let row = crate::OwnedPhysicalRow::new(batch.schema.clone(), row);
272            let view = row.view();
273            encoder.write_row((0..columns.len()).map(|column| {
274                (
275                    view.value_at(column),
276                    column_types.get(column).and_then(Option::as_ref),
277                )
278            }))?;
279        }
280    }
281    encoder.finish()?;
282    Ok(())
283}
284
285fn copy_row_count(rows: usize) -> Result<u64, SQLError> {
286    u64::try_from(rows).map_err(|_| SQLError::Routine {
287        sqlstate: "54000".into(),
288        message: "COPY output row count exceeds u64".into(),
289    })
290}
291
292fn copy_io_error(action: &str, error: std::io::Error) -> SQLError {
293    SQLError::Routine {
294        sqlstate: "58030".into(),
295        message: format!("{action}: {error}"),
296    }
297}
298
299#[cfg(test)]
300mod tests;