sqlload 0.54.0

SQL data loading tooling.
use std::sync::Arc;

use arrow_array::RecordBatch;
use arrow_csv::reader::{Decoder, Format, ReaderBuilder};
use arrow_schema::{DataType, Field, Schema};
use regex::Regex;

use crate::files_source::_helpers::text::bytes_reader::BytesReader;
use crate::files_source::_helpers::text::csv_records::{CsvQuoting, FirstRecord, QuoteTracker};
use crate::files_source::constants::{ENCODING_HINT, MAX_RECORD_BYTES, QUOTE_OFF_BYTE};
use crate::files_source::main::text_charset::text_charset;
use crate::files_source::models::CsvOptions;
use crate::load_naming::main::numbered_header::numbered_header;

/// Reads a CSV file's records in order as text columns, a chunk of bytes at a time.
pub(crate) struct CsvReader {
    bytes: BytesReader,
    /// `None` for a file with no records at all.
    decoder: Option<Decoder>,
    tracker: QuoteTracker,
    pending: Vec<u8>,
    offset: usize,
    at_end: bool,
    /// Whether the decoder was told the file ended (an empty `decode`), which completes a last record without a line break.
    end_sent: bool,
    names: Vec<String>,
    /// The header each column repeats, by position: `None` for a name the header has once, or the first time.
    repeats: Vec<Option<String>>,
    /// The line of the next data record: lines count records, the header and the skipped lines included.
    next_line: u64,
    /// Lines skipped before the header, which the record parsers' line numbers leave out.
    skipped: u64,
}

impl CsvReader {
    /// A reader positioned at the start of `bytes`, with the column names read from the header (or `column_1`, ... without one).
    pub(crate) async fn open(
        bytes: BytesReader,
        options: &CsvOptions,
        batch_rows: usize,
    ) -> Result<Self, String> {
        Self::open_as(bytes, (options, batch_rows), true).await
    }

    /// A reader of whole lines: a header is skipped but never checked, and columns are `column_1`, ... either way.
    pub(crate) async fn open_lines(
        bytes: BytesReader,
        options: &CsvOptions,
        batch_rows: usize,
    ) -> Result<Self, String> {
        Self::open_as(bytes, (options, batch_rows), false).await
    }

    /// A reader positioned after the lines `options` skips in `bytes`, decoded from its encoding; with `named`, its columns are the header's names, checked.
    pub(crate) async fn open_as(
        bytes: BytesReader,
        (options, batch_rows): (&CsvOptions, usize),
        named: bool,
    ) -> Result<Self, String> {
        let quoting = quoting(options)?;
        let skipped = options.skip_rows;
        let mut bytes = bytes.decoding(
            text_charset(&options.encoding)?,
            (skipped, options.encoding_check),
        );
        let mut first_record = FirstRecord::new(quoting, MAX_RECORD_BYTES);
        let mut pending: Vec<u8> = Vec::new();
        let (first, at_end) = loop {
            let chunk = bytes.next_chunk().await?;
            pending.extend_from_slice(&chunk);
            let fed = first_record
                .feed(&chunk)
                .map_err(|error| shifted(&error, skipped))?;
            if let Some(first) = fed {
                break (first, chunk.is_empty());
            }
        };
        let given = options.column_names.as_deref().filter(|_| named);
        let (names, repeats): (Vec<String>, Vec<Option<String>>) =
            column_names(first, options.header && named, given)
                .map_err(|error| shifted(&error, skipped))?
                .into_iter()
                .unzip();
        let decoder = match names.is_empty() {
            true => None,
            false => Some(decoder(&names, (options, quoting), batch_rows)?),
        };
        Ok(Self {
            bytes,
            decoder,
            tracker: QuoteTracker::new(quoting, MAX_RECORD_BYTES),
            pending,
            offset: 0,
            at_end,
            end_sent: false,
            names,
            repeats,
            next_line: 1 + skipped + u64::from(options.header),
            skipped,
        })
    }

    /// The file's columns, in order.
    pub(crate) fn names(&self) -> &[String] {
        &self.names
    }

    /// The header each column repeats, by position (see [`numbered_header`]).
    pub(crate) fn repeats(&self) -> &[Option<String>] {
        &self.repeats
    }

    /// The next batch of records and the line its first row is on; `None` at the end of the file.
    pub(crate) async fn next_batch(&mut self) -> Result<Option<(RecordBatch, u64)>, String> {
        let skipped = self.skipped;
        let parsed = |error: String| shifted(&error, skipped);
        let Some(decoder) = self.decoder.as_mut() else {
            return Ok(None);
        };
        loop {
            if self.offset == self.pending.len() && !self.at_end {
                self.pending = self.bytes.next_chunk().await?;
                self.offset = 0;
                self.at_end = self.pending.is_empty();
            }
            let input = &self.pending[self.offset..];
            if !input.is_empty() {
                let decoded = decoder.decode(input).map_err(csv_error).map_err(parsed)?;
                self.tracker.push(&input[..decoded]).map_err(parsed)?;
                self.offset += decoded;
            } else if self.at_end && !self.end_sent && decoder.capacity() > 0 {
                decoder.decode(&[]).map_err(csv_error).map_err(parsed)?;
                self.end_sent = true;
            }
            let done = self.end_sent && self.offset == self.pending.len();
            if done {
                self.tracker.finish().map_err(parsed)?;
            }
            if decoder.capacity() == 0 || done {
                match decoder.flush().map_err(csv_error).map_err(parsed)? {
                    Some(batch) => {
                        let line = self.next_line;
                        self.next_line += batch.num_rows() as u64;
                        return Ok(Some((batch, line)));
                    }
                    None if done => return Ok(None),
                    None => {}
                }
            }
        }
    }
}

/// An arrow-csv decoder reading every column as text, skipping the header itself so its line numbers count it.
fn decoder(
    names: &[String],
    (options, quoting): (&CsvOptions, CsvQuoting),
    batch_rows: usize,
) -> Result<Decoder, String> {
    let fields: Vec<Field> = names
        .iter()
        .map(|name| Field::new(name, DataType::Utf8, true))
        .collect();
    let format = Format::default()
        .with_header(options.header)
        .with_delimiter(quoting.delimiter)
        .with_quote(quoting.quote.unwrap_or(QUOTE_OFF_BYTE))
        .with_null_regex(null_regex(&options.null_values)?);
    let format = match quoting.escape {
        Some(escape) => format.with_escape(escape),
        None => format,
    };
    Ok(ReaderBuilder::new(Arc::new(Schema::new(fields)))
        .with_format(format)
        .with_batch_size(batch_rows)
        .build_decoder())
}

/// A regex matching exactly the `null_values`; one that never matches when there are none.
fn null_regex(null_values: &[String]) -> Result<Regex, String> {
    let pattern = match null_values.is_empty() {
        true => r"[^\s\S]".to_owned(),
        false => {
            let alternatives: Vec<String> = null_values
                .iter()
                .map(|value| regex::escape(value))
                .collect();
            format!(r"\A(?:{})\z", alternatives.join("|"))
        }
    };
    Regex::new(&pattern).map_err(|error| format!("csv null_values: {error}"))
}

/// `options`' delimiter, quote and escape as bytes; arrow-csv can't turn quoting off, so with `quote = false` it gets [`QUOTE_OFF_BYTE`] as the quote.
fn quoting(options: &CsvOptions) -> Result<CsvQuoting, String> {
    Ok(CsvQuoting {
        delimiter: ascii(options.delimiter, "delimiter")?,
        quote: options
            .quote
            .character()
            .map(|quote| ascii(quote, "quote"))
            .transpose()?,
        escape: options
            .escape
            .map(|escape| ascii(escape, "escape"))
            .transpose()?,
    })
}

fn ascii(character: char, setting: &str) -> Result<u8, String> {
    match u8::try_from(character) {
        Ok(byte) if byte.is_ascii() => Ok(byte),
        _ => Err(format!(
            "csv {setting} must be one ASCII character, not {character:?}"
        )),
    }
}

/// The header's names, checked, a repeated one numbered (see [`numbered_header`]); or for a file without a header, `given` (one per field of the first record) or `column_1`, ...
fn column_names(
    first: Vec<String>,
    header: bool,
    given: Option<&[String]>,
) -> Result<Vec<(String, Option<String>)>, String> {
    let plain = |names: Vec<String>| names.into_iter().map(|name| (name, None)).collect();
    match (header, given) {
        (false, Some(given)) if first.len() == given.len() => return Ok(plain(given.to_vec())),
        (false, Some(given)) if !first.is_empty() => {
            return Err(format!(
                "line 1: the file has {} columns, but column_names names {}; name each column",
                first.len(),
                given.len()
            ));
        }
        (false, _) => {
            return Ok(plain(
                (1..=first.len())
                    .map(|position| format!("column_{position}"))
                    .collect(),
            ));
        }
        (true, _) => {}
    }
    for (position, name) in first.iter().enumerate() {
        if name.is_empty() {
            return Err(format!("line 1: header field {} is empty", position + 1));
        }
    }
    Ok(numbered_header(&first))
}

/// arrow-csv's error without its prefix; invalid UTF-8 says which encoding setting fixes it.
fn csv_error(error: arrow_schema::ArrowError) -> String {
    let text = error.to_string();
    let text = text.strip_prefix("Csv error: ").unwrap_or(&text);
    let invalid = text
        .strip_prefix("Encountered invalid UTF-8 data for line ")
        .and_then(|rest| rest.split_once(" and field "));
    match invalid {
        Some((line, field)) => {
            format!("line {line}, field {field}: the bytes are not valid UTF-8; {ENCODING_HINT}")
        }
        None => text.to_owned(),
    }
}

/// `error` with each `line N` it names moved down by the `skipped` lines the parser never saw.
fn shifted(error: &str, skipped: u64) -> String {
    if skipped == 0 {
        return error.to_owned();
    }
    let Ok(lines) = Regex::new(r"\bline (\d+)") else {
        return error.to_owned();
    };
    lines
        .replace_all(error, |found: &regex::Captures<'_>| {
            let line = found[1].parse::<u64>().map_or(0, |line| line + skipped);
            format!("line {line}")
        })
        .into_owned()
}