Skip to main content

uqa_sql/copy/
codec.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! `PostgreSQL` COPY text and CSV stream codecs.
8
9use super::{CopyFormat, CopyHeader, CopyInputField, CopyOptions};
10use crate::{SQLError, SQLResult};
11
12/// Decode a complete text or CSV COPY stream into rows.
13pub fn decode_copy_input(
14    bytes: &[u8],
15    options: &CopyOptions,
16    expected_columns: &[String],
17) -> Result<Vec<Vec<CopyInputField>>, SQLError> {
18    if options.format == CopyFormat::Binary {
19        return Err(SQLError::Unsupported(
20            "binary COPY format is not implemented".into(),
21        ));
22    }
23    let input = copy_utf8(bytes)?;
24    let mut rows = match options.format {
25        CopyFormat::Text => decode_text_rows(input, options)?,
26        CopyFormat::Csv => decode_csv_rows(input, options)?,
27        CopyFormat::Binary => unreachable!(),
28    };
29    if options.header != CopyHeader::False {
30        if rows.is_empty() {
31            return Ok(Vec::new());
32        }
33        let header = rows.remove(0);
34        if options.header == CopyHeader::Match {
35            let actual = header
36                .into_iter()
37                .map(|field| field.unwrap_or_default())
38                .collect::<Vec<_>>();
39            if actual != expected_columns {
40                return Err(copy_data_error(
41                    "22P04",
42                    format!(
43                        "wrong number of fields in header line: expected {expected_columns:?}, got {actual:?}"
44                    ),
45                    1,
46                ));
47            }
48        }
49    }
50    let first_data_line = usize::from(options.header != CopyHeader::False) + 1;
51    for (index, row) in rows.iter().enumerate() {
52        if row.len() > expected_columns.len() {
53            return Err(copy_data_error(
54                "22P04",
55                "extra data after last expected column",
56                index + first_data_line,
57            ));
58        }
59        if row.len() < expected_columns.len() {
60            return Err(copy_data_error(
61                "22P04",
62                format!(
63                    "missing data for column \"{}\"",
64                    expected_columns[row.len()]
65                ),
66                index + first_data_line,
67            ));
68        }
69    }
70    Ok(rows)
71}
72
73fn decode_text_rows(
74    input: &str,
75    options: &CopyOptions,
76) -> Result<Vec<Vec<CopyInputField>>, SQLError> {
77    let mut rows = Vec::new();
78    for (line_index, raw) in logical_text_lines(input).into_iter().enumerate() {
79        if raw == "\\." {
80            break;
81        }
82        rows.push(decode_text_row(raw, options, line_index + 1)?);
83    }
84    Ok(rows)
85}
86
87fn decode_text_row(
88    raw: &str,
89    options: &CopyOptions,
90    line: usize,
91) -> Result<Vec<CopyInputField>, SQLError> {
92    let bytes = raw.as_bytes();
93    let mut fields = Vec::new();
94    let mut field_start = 0usize;
95    let mut index = 0usize;
96    while index < bytes.len() {
97        if bytes[index] == options.delimiter {
98            fields.push(decode_text_input_field(
99                &raw[field_start..index],
100                options,
101                line,
102            )?);
103            index += 1;
104            field_start = index;
105            continue;
106        }
107        if bytes[index] != b'\\' {
108            index += 1;
109            continue;
110        }
111        index += 1;
112        if index == bytes.len() {
113            return Err(copy_data_error(
114                "22P04",
115                "unterminated COPY data escape",
116                line,
117            ));
118        }
119        match bytes[index] {
120            b'x' => {
121                index += 1;
122                for _ in 0..2 {
123                    if index >= bytes.len() || hex_value(bytes[index]).is_none() {
124                        break;
125                    }
126                    index += 1;
127                }
128            }
129            b'0'..=b'7' => {
130                index += 1;
131                for _ in 0..2 {
132                    if index >= bytes.len() || !(b'0'..=b'7').contains(&bytes[index]) {
133                        break;
134                    }
135                    index += 1;
136                }
137            }
138            _ => index += 1,
139        }
140    }
141    fields.push(decode_text_input_field(&raw[field_start..], options, line)?);
142    Ok(fields)
143}
144
145fn decode_text_input_field(
146    raw: &str,
147    options: &CopyOptions,
148    line: usize,
149) -> Result<CopyInputField, SQLError> {
150    if raw == options.null {
151        Ok(None)
152    } else {
153        decode_text_field(raw, line).map(Some)
154    }
155}
156
157fn logical_text_lines(input: &str) -> Vec<&str> {
158    if input.is_empty() {
159        return Vec::new();
160    }
161    let mut lines = input.split('\n').collect::<Vec<_>>();
162    if input.ends_with('\n') {
163        lines.pop();
164    }
165    for line in &mut lines {
166        if let Some(without_cr) = line.strip_suffix('\r') {
167            *line = without_cr;
168        }
169    }
170    lines
171}
172
173fn decode_text_field(raw: &str, line: usize) -> Result<String, SQLError> {
174    let bytes = raw.as_bytes();
175    let mut output = Vec::with_capacity(bytes.len());
176    let mut index = 0;
177    while index < bytes.len() {
178        if bytes[index] != b'\\' {
179            output.push(bytes[index]);
180            index += 1;
181            continue;
182        }
183        index += 1;
184        if index == bytes.len() {
185            return Err(copy_data_error(
186                "22P04",
187                "unterminated COPY data escape",
188                line,
189            ));
190        }
191        match bytes[index] {
192            b'b' => output.push(0x08),
193            b'f' => output.push(0x0c),
194            b'n' => output.push(b'\n'),
195            b'r' => output.push(b'\r'),
196            b't' => output.push(b'\t'),
197            b'v' => output.push(0x0b),
198            b'x' => {
199                let mut value = 0u8;
200                let mut digits = 0usize;
201                while digits < 2 && index + 1 < bytes.len() {
202                    let Some(nibble) = hex_value(bytes[index + 1]) else {
203                        break;
204                    };
205                    value = value * 16 + nibble;
206                    index += 1;
207                    digits += 1;
208                }
209                if digits == 0 {
210                    output.push(b'x');
211                } else {
212                    output.push(value);
213                }
214            }
215            digit @ b'0'..=b'7' => {
216                let mut value = digit - b'0';
217                let mut digits = 1usize;
218                while digits < 3 && index + 1 < bytes.len() {
219                    let next = bytes[index + 1];
220                    if !(b'0'..=b'7').contains(&next) {
221                        break;
222                    }
223                    value = value.wrapping_mul(8).wrapping_add(next - b'0');
224                    index += 1;
225                    digits += 1;
226                }
227                output.push(value);
228            }
229            escaped => output.push(escaped),
230        }
231        index += 1;
232    }
233    if output.contains(&0) {
234        return Err(copy_encoding_error(Some(line)));
235    }
236    String::from_utf8(output).map_err(|_| copy_encoding_error(Some(line)))
237}
238
239fn hex_value(byte: u8) -> Option<u8> {
240    match byte {
241        b'0'..=b'9' => Some(byte - b'0'),
242        b'a'..=b'f' => Some(byte - b'a' + 10),
243        b'A'..=b'F' => Some(byte - b'A' + 10),
244        _ => None,
245    }
246}
247
248fn decode_csv_rows(
249    input: &str,
250    options: &CopyOptions,
251) -> Result<Vec<Vec<CopyInputField>>, SQLError> {
252    let bytes = input.as_bytes();
253    let mut rows = Vec::new();
254    let mut row = Vec::new();
255    let mut field = Vec::new();
256    let mut quoted = false;
257    let mut in_quotes = false;
258    let mut after_quote = false;
259    let mut index = 0usize;
260    let mut line = 1usize;
261    while index < bytes.len() {
262        let byte = bytes[index];
263        if in_quotes {
264            if byte == options.escape {
265                if index + 1 < bytes.len()
266                    && matches!(bytes[index + 1], next if next == options.quote || next == options.escape)
267                {
268                    field.push(bytes[index + 1]);
269                    index += 2;
270                    continue;
271                }
272                if options.escape != options.quote {
273                    field.push(byte);
274                    index += 1;
275                    continue;
276                }
277            }
278            if byte == options.quote {
279                in_quotes = false;
280                after_quote = true;
281                index += 1;
282                continue;
283            }
284            if byte == b'\n' {
285                line += 1;
286            }
287            field.push(byte);
288            index += 1;
289            continue;
290        }
291        if after_quote && !matches!(byte, b'\n' | b'\r') && byte != options.delimiter {
292            field.push(byte);
293            index += 1;
294            continue;
295        }
296        if byte == options.delimiter {
297            row.push(csv_field(&field, quoted, options)?);
298            field.clear();
299            quoted = false;
300            after_quote = false;
301            index += 1;
302            continue;
303        }
304        if byte == b'\n' || byte == b'\r' {
305            if byte == b'\r' && index + 1 < bytes.len() && bytes[index + 1] == b'\n' {
306                index += 1;
307            }
308            row.push(csv_field(&field, quoted, options)?);
309            if row.len() == 1 && row[0].as_deref() == Some("\\.") && !quoted {
310                return Ok(rows);
311            }
312            rows.push(std::mem::take(&mut row));
313            field.clear();
314            quoted = false;
315            after_quote = false;
316            index += 1;
317            line += 1;
318            continue;
319        }
320        if byte == options.quote && field.is_empty() && !after_quote {
321            quoted = true;
322            in_quotes = true;
323            index += 1;
324            continue;
325        }
326        field.push(byte);
327        index += 1;
328    }
329    if in_quotes {
330        return Err(copy_data_error(
331            "22P04",
332            "unterminated CSV quoted field",
333            line,
334        ));
335    }
336    if !field.is_empty() || !row.is_empty() || quoted || after_quote {
337        row.push(csv_field(&field, quoted, options)?);
338        if !(row.len() == 1 && row[0].as_deref() == Some("\\.") && !quoted) {
339            rows.push(row);
340        }
341    }
342    Ok(rows)
343}
344
345fn csv_field(
346    field: &[u8],
347    quoted: bool,
348    options: &CopyOptions,
349) -> Result<CopyInputField, SQLError> {
350    let text = copy_utf8(field)?;
351    if !quoted && text == options.null {
352        Ok(None)
353    } else {
354        Ok(Some(text.to_string()))
355    }
356}
357
358fn copy_data_error(sqlstate: &str, message: impl Into<String>, line: usize) -> SQLError {
359    SQLError::Routine {
360        sqlstate: sqlstate.into(),
361        message: format!("{}\nCONTEXT: COPY data, line {line}", message.into()),
362    }
363}
364
365fn copy_utf8(bytes: &[u8]) -> Result<&str, SQLError> {
366    if bytes.contains(&0) {
367        return Err(copy_encoding_error(None));
368    }
369    std::str::from_utf8(bytes).map_err(|_| copy_encoding_error(None))
370}
371
372fn copy_encoding_error(line: Option<usize>) -> SQLError {
373    let context = line.map_or_else(String::new, |line| format!(" at line {line}"));
374    SQLError::Routine {
375        sqlstate: "22021".into(),
376        message: format!("invalid byte sequence for encoding \"UTF8\" in COPY data{context}"),
377    }
378}
379
380/// Encode one SQL result using `PostgreSQL` COPY text or CSV representation.
381pub fn encode_copy_result(result: &SQLResult, options: &CopyOptions) -> Result<Vec<u8>, SQLError> {
382    let mut output = Vec::new();
383    super::output::write_copy_result(
384        result,
385        options,
386        &mut output,
387        None,
388        &uqa_core::memory::ProductionControl::uncontrolled(),
389    )?;
390    Ok(output)
391}
392
393/// Encode one SQL result while resolving catalog-backed `reg*` text output through the engine hook.
394pub fn encode_copy_result_with_engine(
395    result: &SQLResult,
396    options: &CopyOptions,
397    engine: &dyn crate::expr::EngineHook,
398) -> Result<Vec<u8>, SQLError> {
399    let mut output = Vec::new();
400    super::output::write_copy_result(
401        result,
402        options,
403        &mut output,
404        Some(engine),
405        &uqa_core::memory::ProductionControl::uncontrolled(),
406    )?;
407    Ok(output)
408}