Skip to main content

uqa_execution/
copy.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Execute COPY byte streams through the ordinary INSERT and SELECT statement paths.
8use crate::{query::output::QueryRows, statement::plan_executor::StatementOutput};
9use std::io::{Read, Write};
10use uqa_core::{
11    memory::{MemoryBudget, ProductionControl},
12    CancellationToken, Value,
13};
14use uqa_sql::{
15    ast::{Expr, InsertStmt, ReturningAliases, Statement},
16    catalog::security::table::TableAclPrivilege,
17    copy::stream::CopyCatalog,
18    copy::{
19        compile_copy, decode_copy_input, write_copy_result, CopyDirection, CopyEncoder, CopyTarget,
20    },
21    expr::EngineHook,
22    SQLError, SQLParam, SQLResult,
23};
24pub trait CopyPrivileges {
25    fn ensure_any_column(&self, table: &str, privilege: TableAclPrivilege) -> Result<(), SQLError>;
26    fn ensure_column(
27        &self,
28        table: &str,
29        column: &str,
30        privilege: TableAclPrivilege,
31    ) -> Result<(), SQLError>;
32}
33pub trait CopyStatements {
34    fn execute_statement(
35        &self,
36        statement: Statement,
37        params: &[SQLParam],
38    ) -> Result<SQLResult, SQLError>;
39    fn execute_text_with_output(
40        &self,
41        text: &str,
42        params: &[SQLParam],
43        consume: &mut dyn FnMut(StatementOutput) -> Result<(), SQLError>,
44    ) -> Result<(), SQLError>;
45}
46pub struct CopyExecutionContext<'a> {
47    pub catalog: &'a dyn CopyCatalog,
48    pub privileges: &'a dyn CopyPrivileges,
49    pub statements: &'a dyn CopyStatements,
50    pub output: &'a dyn EngineHook,
51    pub memory: &'a dyn crate::query::runtime::QueryMemorySettings,
52    pub cancellation: &'a CancellationToken,
53}
54pub fn copy_from(
55    context: &CopyExecutionContext<'_>,
56    statement: &str,
57    input: &mut impl Read,
58) -> Result<u64, SQLError> {
59    let copy = compile_copy(statement)?;
60    uqa_sql::copy::stream::validate_stream(&copy, CopyDirection::From)?;
61    let CopyTarget::Relation {
62        name: relation,
63        qualifier,
64        columns: requested_columns,
65    } = &copy.target
66    else {
67        return Err(SQLError::Routine {
68            sqlstate: "42601".into(),
69            message: "COPY FROM requires a relation target".into(),
70        });
71    };
72    let (canonical, columns) = uqa_sql::copy::stream::relation_columns(
73        context.catalog,
74        relation,
75        qualifier,
76        requested_columns,
77        false,
78    )?;
79    if columns.is_empty() {
80        context
81            .privileges
82            .ensure_any_column(&canonical, TableAclPrivilege::Insert)?;
83    } else {
84        for column in &columns {
85            context
86                .privileges
87                .ensure_column(&canonical, column, TableAclPrivilege::Insert)?;
88        }
89    }
90    let bytes = read_copy_input(input, context.cancellation)?;
91    let rows = decode_copy_input(&bytes, &copy.options, &columns)?;
92    if rows.is_empty() {
93        return Ok(0);
94    }
95    if columns.is_empty() {
96        return Err(SQLError::Unsupported(
97            "COPY FROM for a zero-column relation is not implemented".into(),
98        ));
99    }
100    let mut params = Vec::with_capacity(rows.len().saturating_mul(columns.len()));
101    let mut parameter = 1usize;
102    let mut insert_rows = Vec::with_capacity(rows.len());
103    for row in rows {
104        let mut values = Vec::with_capacity(row.len());
105        for field in row {
106            values.push(Expr::Param(parameter));
107            parameter = parameter.checked_add(1).ok_or_else(|| SQLError::Routine {
108                sqlstate: "54000".into(),
109                message: "COPY input has too many fields".into(),
110            })?;
111            params.push(SQLParam::Scalar(match field {
112                Some(value) => Value::Str(value),
113                None => Value::Null,
114            }));
115        }
116        insert_rows.push(values);
117    }
118    let result = context.statements.execute_statement(
119        Statement::Insert(InsertStmt {
120            table: relation.clone(),
121            target_relation_bound: false,
122            target_qualifier: qualifier.clone(),
123            target_alias: None,
124            include_descendants: true,
125            columns: columns.into_iter().map(Into::into).collect(),
126            // COPY FROM writes the values its input supplies, also to a GENERATED ALWAYS identity column.
127            overriding: Some(uqa_sql::ast::OverridingKind::SystemValue),
128            with: Vec::new(),
129            rows: insert_rows,
130            select_source: None,
131            on_conflict: None,
132            returning: Vec::new(),
133            returning_aliases: ReturningAliases::default(),
134        }),
135        &params,
136    )?;
137    Ok(result.affected_rows)
138}
139
140pub fn copy_to(
141    context: &CopyExecutionContext<'_>,
142    statement: &str,
143    output: &mut impl Write,
144) -> Result<u64, SQLError> {
145    let copy = compile_copy(statement)?;
146    uqa_sql::copy::stream::validate_stream(&copy, CopyDirection::To)?;
147    let query = match &copy.target {
148        CopyTarget::Relation {
149            name: relation,
150            qualifier,
151            columns: requested_columns,
152        } => {
153            let (canonical, columns) = uqa_sql::copy::stream::relation_columns(
154                context.catalog,
155                relation,
156                qualifier,
157                requested_columns,
158                true,
159            )?;
160            if columns.is_empty() {
161                context
162                    .privileges
163                    .ensure_any_column(&canonical, TableAclPrivilege::Select)?;
164            } else {
165                for column in &columns {
166                    context.privileges.ensure_column(
167                        &canonical,
168                        column,
169                        TableAclPrivilege::Select,
170                    )?;
171                }
172            }
173            let projection = columns
174                .iter()
175                .map(|column| uqa_sql::expr::quote_ident(column))
176                .collect::<Vec<_>>()
177                .join(", ");
178            format!("SELECT {projection} FROM ONLY {relation}")
179        }
180        CopyTarget::Query(query) => query.clone(),
181    };
182    let mut row_count = 0;
183    context
184        .statements
185        .execute_text_with_output(&query, &[], &mut |result| {
186            let memory = MemoryBudget::new(context.memory.work_mem_bytes()?);
187            let control =
188                ProductionControl::new(&memory, context.cancellation, context.cancellation);
189            row_count = write_statement_output(
190                result,
191                &copy.options,
192                output,
193                Some(context.output),
194                &control,
195            )?;
196            Ok(())
197        })?;
198    Ok(row_count)
199}
200
201pub(crate) fn write_statement_output(
202    result: StatementOutput,
203    options: &uqa_sql::copy::CopyOptions,
204    output: &mut impl Write,
205    engine: Option<&dyn EngineHook>,
206    control: &ProductionControl<'_>,
207) -> Result<u64, SQLError> {
208    match result {
209        StatementOutput::Command(result) => {
210            let count = copy_row_count(result.rows.len())?;
211            write_copy_result(&result, options, output, engine, control)?;
212            Ok(count)
213        }
214        StatementOutput::Mutation(result) => {
215            let count = copy_row_count(result.row_count())?;
216            let schema = result.schema.clone();
217            write_copy_batches(
218                schema.columns(),
219                schema.column_types(),
220                result.into_batches(),
221                options,
222                output,
223                engine,
224                control,
225            )?;
226            Ok(count)
227        }
228        StatementOutput::Query(result) => {
229            let QueryRows::SharedSpill(rows) = result.rows else {
230                return Err(SQLError::Internal(
231                    "COPY query returned unbounded rows".into(),
232                ));
233            };
234            let count = copy_row_count(rows.rows())?;
235            let rows = rows
236                .into_reader()
237                .map_err(crate::physical::physical_exec_error)?;
238            write_copy_batches(
239                &result.columns,
240                &result.column_types,
241                rows.map(|batch| batch.map_err(crate::physical::physical_exec_error)),
242                options,
243                output,
244                engine,
245                control,
246            )?;
247            Ok(count)
248        }
249    }
250}
251
252fn write_copy_batches(
253    columns: &[String],
254    column_types: &[Option<uqa_sql::ast::ColumnType>],
255    mut batches: impl Iterator<Item = Result<crate::Batch, SQLError>>,
256    options: &uqa_sql::copy::CopyOptions,
257    output: &mut impl Write,
258    engine: Option<&dyn EngineHook>,
259    control: &ProductionControl<'_>,
260) -> Result<(), SQLError> {
261    let mut encoder = CopyEncoder::new(options, output, engine, *control)?;
262    encoder.write_header(columns)?;
263    loop {
264        control.check()?;
265        let Some(batch) = batches.next() else { break };
266        let batch = batch?;
267        for row in batch.rows {
268            let row = crate::OwnedPhysicalRow::new(batch.schema.clone(), row);
269            let view = row.view();
270            encoder.write_row((0..columns.len()).map(|column| {
271                (
272                    view.value_at(column),
273                    column_types.get(column).and_then(Option::as_ref),
274                )
275            }))?;
276        }
277    }
278    encoder.finish()?;
279    Ok(())
280}
281
282fn copy_row_count(rows: usize) -> Result<u64, SQLError> {
283    u64::try_from(rows).map_err(|_| SQLError::Routine {
284        sqlstate: "54000".into(),
285        message: "COPY output row count exceeds u64".into(),
286    })
287}
288
289fn copy_io_error(action: &str, error: std::io::Error) -> SQLError {
290    SQLError::Routine {
291        sqlstate: "58030".into(),
292        message: format!("{action}: {error}"),
293    }
294}
295
296/// The complete input still precedes insertion, but a continuing source cannot defer cancellation
297/// until EOF. Keep this check in the stream owner so every language uses the same boundary.
298fn read_copy_input(
299    input: &mut impl Read,
300    cancellation: &CancellationToken,
301) -> Result<Vec<u8>, SQLError> {
302    let check = || -> Result<(), SQLError> {
303        cancellation.check()?;
304        Ok(uqa_sql::expr::check_statement_cancellation()?)
305    };
306    let mut bytes = Vec::new();
307    let mut buffer = [0; 8192];
308    loop {
309        check()?;
310        let count = match input.read(&mut buffer) {
311            Ok(count) => count,
312            Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
313            Err(error) => return Err(copy_io_error("read COPY FROM stream", error)),
314        };
315        check()?;
316        if count == 0 {
317            return Ok(bytes);
318        }
319        let chunk = buffer.get(..count).ok_or_else(|| {
320            copy_io_error(
321                "read COPY FROM stream",
322                std::io::Error::other("reader returned a count larger than its buffer"),
323            )
324        })?;
325        bytes
326            .try_reserve(chunk.len())
327            .map_err(|_| SQLError::Routine {
328                sqlstate: "53200".into(),
329                message: "out of memory while reading COPY input".into(),
330            })?;
331        bytes.extend_from_slice(chunk);
332    }
333}
334
335#[cfg(test)]
336mod tests;