use super::import::ImportRequest;
use crate::error::{Failure, Result};
use std::path::Path;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Rejection {
pub row: u64,
pub reason: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Batch {
pub start: u64,
pub end: u64,
pub rows: Vec<Vec<String>>,
pub positions: Vec<u64>,
}
#[derive(Debug, Clone, Default)]
pub struct Reading {
pub columns: Vec<String>,
pub batches: Vec<Batch>,
pub rejected: Vec<Rejection>,
pub rows_read: u64,
}
pub struct Batches {
reader: csv::Reader<std::fs::File>,
batch_size: usize,
resume_from: u64,
width: usize,
columns: Vec<String>,
offset: u64,
start: u64,
rejected: Vec<Rejection>,
rows_read: u64,
}
impl Batches {
pub fn open(file: &Path, request: &ImportRequest) -> Result<Self> {
if request.batch_size == 0 {
return Err(Failure::usage(
"datamove.empty_batch",
"a batch of no rows would never commit anything",
)
.with_hint("ask for at least one row per batch"));
}
let mut reader = csv::ReaderBuilder::new()
.delimiter(request.separator)
.has_headers(request.header)
.flexible(true)
.from_path(file)
.map_err(|error| unreadable(file, &error))?;
let columns: Vec<String> = if request.header {
let named = reader.headers().map_err(|error| unreadable(file, &error))?;
named.iter().map(ToOwned::to_owned).collect()
} else {
Vec::new()
};
Ok(Self {
reader,
batch_size: request.batch_size,
resume_from: request.resume_from,
width: columns.len(),
columns,
offset: 0,
start: request.resume_from,
rejected: Vec::new(),
rows_read: 0,
})
}
#[must_use]
pub fn columns(&self) -> &[String] {
&self.columns
}
#[must_use]
pub const fn rows_read(&self) -> u64 {
self.rows_read
}
pub fn take_rejected(&mut self) -> Vec<Rejection> {
std::mem::take(&mut self.rejected)
}
pub fn next_batch(&mut self) -> Option<Batch> {
let mut rows = Vec::with_capacity(self.batch_size);
let mut positions = Vec::with_capacity(self.batch_size);
while let Some(record) = self.reader.records().next() {
let position = self.offset;
self.offset += 1;
if position < self.resume_from {
continue;
}
self.rows_read += 1;
match fields_of(record, self.width) {
Ok(fields) => {
rows.push(fields);
positions.push(position);
}
Err(reason) => {
self.rejected.push(Rejection {
row: position,
reason,
});
continue;
}
}
if rows.len() >= self.batch_size {
let end = position + 1;
let start = self.start;
self.start = end;
return Some(Batch {
start,
end,
rows,
positions,
});
}
}
if rows.is_empty() {
return None;
}
let start = self.start;
self.start = self.offset;
Some(Batch {
start,
end: self.offset,
rows,
positions,
})
}
}
pub fn read(file: &Path, request: &ImportRequest) -> Result<Reading> {
let mut batches = Batches::open(file, request)?;
let mut reading = Reading::default();
while let Some(batch) = batches.next_batch() {
reading.batches.push(batch);
reading.rejected.extend(batches.take_rejected());
}
reading.rejected.extend(batches.take_rejected());
reading.rows_read = batches.rows_read();
reading.columns = batches.columns().to_vec();
Ok(reading)
}
fn fields_of(
record: csv::Result<csv::StringRecord>,
width: usize,
) -> std::result::Result<Vec<String>, String> {
let record = record.map_err(|error| format!("the line could not be read: {error}"))?;
if width > 0 && record.len() != width {
return Err(format!(
"the line carries {} fields where the header names {width}",
record.len()
));
}
Ok(record.iter().map(ToOwned::to_owned).collect())
}
fn unreadable(file: &Path, cause: &impl std::fmt::Display) -> Failure {
Failure::config(
"datamove.file_unreadable",
format!("`{}` could not be read", file.display()),
)
.with_detail("detail", file.display().to_string())
.with_hint("this process opens the file, so the path is read from here and not from the server")
.with_cause(cause.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::PathBuf;
fn request(file: &Path, batch_size: usize) -> ImportRequest {
ImportRequest {
file: file.to_path_buf(),
table: "customers".to_owned(),
database: None,
batch_size,
resume_from: 0,
separator: b',',
header: true,
stop_on_refusal: false,
}
}
fn write_file(content: &str) -> (tempfile::TempDir, PathBuf) {
let directory = tempfile::tempdir().expect("scratch directory");
let path = directory.path().join("rows.csv");
std::fs::write(&path, content).expect("writes");
(directory, path)
}
#[test]
fn a_long_file_is_walked_without_holding_more_than_one_batch() {
use std::fmt::Write as _;
let mut content = String::from("id,name\n");
for row in 0..5_000 {
writeln!(content, "{row},name-{row}").expect("a string always takes a write");
}
let (_guard, path) = write_file(&content);
let mut batches = Batches::open(&path, &request(&path, 100)).expect("opens");
let mut seen = 0_u64;
let mut rounds = 0_usize;
while let Some(batch) = batches.next_batch() {
assert!(
batch.rows.len() <= 100,
"a batch carried {} rows against a ceiling of 100",
batch.rows.len()
);
seen += batch.rows.len() as u64;
rounds += 1;
}
assert_eq!(seen, 5_000, "every row was walked");
assert_eq!(rounds, 50, "five thousand rows in batches of a hundred");
assert_eq!(batches.rows_read(), 5_000);
}
#[test]
fn the_rows_are_cut_into_batches_that_each_carry_their_offset() {
let (_guard, path) = write_file("id\n1\n2\n3\n4\n5\n");
let reading = read(&path, &request(&path, 2)).expect("reads");
assert_eq!(reading.batches.len(), 3);
assert_eq!(reading.batches[0].start, 0);
assert_eq!(reading.batches[2].rows.len(), 1);
let ends: Vec<u64> = reading.batches.iter().map(|batch| batch.end).collect();
assert_eq!(ends, vec![2, 4, 5]);
}
#[test]
fn a_resumed_load_starts_where_the_last_one_stopped() {
let (_guard, path) = write_file("id\n1\n2\n3\n4\n");
let mut asked = request(&path, 10);
asked.resume_from = 2;
let reading = read(&path, &asked).expect("reads");
assert_eq!(reading.rows_read, 2, "the rows already loaded are skipped");
assert_eq!(reading.batches.len(), 1);
assert_eq!(reading.batches[0].start, 2);
assert_eq!(reading.batches[0].end, 4);
assert_eq!(
reading.batches[0].rows,
vec![vec!["3".to_owned()], vec!["4".to_owned()]]
);
}
#[test]
fn every_refused_row_is_named_and_the_rest_still_load() {
let (_guard, path) = write_file("id,name\n1\n2,bruno\n3\n4,dora\n");
let reading = read(&path, &request(&path, 10)).expect("reads");
let positions: Vec<u64> = reading.rejected.iter().map(|entry| entry.row).collect();
assert_eq!(positions, vec![0, 2]);
for entry in &reading.rejected {
assert!(entry.reason.contains("fields"), "a reason, not a count");
}
assert_eq!(reading.batches[0].rows.len(), 2, "the rest still load");
}
#[test]
fn a_file_with_no_header_takes_every_line_as_a_row() {
let (_guard, path) = write_file("1,ana\n2,bruno\n");
let mut asked = request(&path, 10);
asked.header = false;
let reading = read(&path, &asked).expect("reads");
assert!(reading.columns.is_empty());
assert_eq!(reading.rows_read, 2);
}
#[test]
fn another_separator_is_read_the_same_way() {
for (content, separator) in [("id,name\n1,ana\n", b','), ("id\tname\n1\tana\n", b'\t')] {
let (_guard, path) = write_file(content);
let mut asked = request(&path, 10);
asked.separator = separator;
let reading = read(&path, &asked).expect("reads");
assert_eq!(reading.columns, vec!["id".to_owned(), "name".to_owned()]);
assert_eq!(
reading.batches[0].rows[0],
vec!["1".to_owned(), "ana".to_owned()]
);
assert!(reading.rejected.is_empty());
}
}
#[test]
fn a_file_that_is_not_there_is_a_configuration_failure_of_this_side() {
let absent = Path::new("/nowhere/at/all/rows.csv");
let failure = read(absent, &request(absent, 10)).expect_err("must refuse");
assert_eq!(failure.code(), "datamove.file_unreadable");
let hint = failure.hint().unwrap_or_default();
assert!(hint.contains("not from the server"), "{hint}");
}
#[test]
fn a_batch_of_no_rows_is_refused_rather_than_looping_forever() {
let (_guard, path) = write_file("id\n1\n");
let failure = read(&path, &request(&path, 0)).expect_err("must refuse");
assert_eq!(failure.code(), "datamove.empty_batch");
assert_eq!(failure.exit_number().code(), 2);
}
}