use super::import_rows::{self, Batch, Rejection};
use crate::conn::{map_driver_error, OrFailure};
use crate::error::{Failure, Result};
use crate::sql::ident;
use mysql_async::prelude::Queryable;
use mysql_async::{Conn, Params, Value as Wire};
use std::path::PathBuf;
use tokio_util::sync::CancellationToken;
#[derive(Debug, Clone)]
pub struct ImportRequest {
pub file: PathBuf,
pub table: String,
pub database: Option<String>,
pub batch_size: usize,
pub resume_from: u64,
pub separator: u8,
pub header: bool,
pub stop_on_refusal: bool,
}
#[derive(Debug, Clone)]
pub struct ImportReport {
pub rows_read: u64,
pub rows_written: u64,
pub rejected: Vec<Rejection>,
pub offset: u64,
pub columns: Vec<String>,
}
pub fn statement(table: &str, columns: &[String], rows: usize, width: usize) -> Result<String> {
let quoted = ident::quote(table)?;
let names = if columns.is_empty() {
String::new()
} else {
let quoted_columns = columns
.iter()
.map(|name| ident::quote(name))
.collect::<Result<Vec<_>>>()?;
format!(" ({})", quoted_columns.join(", "))
};
let placeholders = vec!["?"; width].join(", ");
let tuples = vec![format!("({placeholders})"); rows].join(", ");
Ok(format!("INSERT INTO {quoted}{names} VALUES {tuples}"))
}
pub async fn run(
connection: &mut Conn,
request: &ImportRequest,
token: &CancellationToken,
) -> Result<ImportReport> {
let mut batches = import_rows::Batches::open(&request.file, request)?;
let columns: Vec<String> = batches.columns().to_vec();
let mut report = ImportReport {
rows_read: 0,
rows_written: 0,
rejected: Vec::new(),
offset: request.resume_from,
columns: columns.clone(),
};
while let Some(batch) = batches.next_batch() {
for rejection in batches.take_rejected() {
crate::i18n::report::load_row_rejected(rejection.row, &rejection.reason);
if request.stop_on_refusal {
report.rows_read = batches.rows_read();
let failure = refused(&rejection);
report.rejected.push(rejection);
return Err(failure);
}
report.rejected.push(rejection);
}
if token.is_cancelled() {
report.rows_read = batches.rows_read();
return Err(stopped(&report));
}
commit_batch(connection, request, &columns, &batch, &mut report).await?;
}
for rejection in batches.take_rejected() {
crate::i18n::report::load_row_rejected(rejection.row, &rejection.reason);
if request.stop_on_refusal {
report.rows_read = batches.rows_read();
let failure = refused(&rejection);
report.rejected.push(rejection);
return Err(failure);
}
report.rejected.push(rejection);
}
report.rows_read = batches.rows_read();
crate::i18n::report::load_finished(report.rows_written, report.rejected.len() as u64);
Ok(report)
}
fn stopped(report: &ImportReport) -> Failure {
Failure::interrupted(
"interrupted.during_load",
format!(
"the invocation was stopped after {} rows; resume from offset {}",
report.rows_written, report.offset
),
)
.with_detail("detail", report.offset.to_string())
}
async fn commit_batch(
connection: &mut Conn,
request: &ImportRequest,
columns: &[String],
batch: &Batch,
report: &mut ImportReport,
) -> Result<()> {
let Some(width) = batch.rows.first().map(Vec::len) else {
return Ok(());
};
let text = statement(&request.table, columns, batch.rows.len(), width)?;
let values: Vec<Wire> = batch
.rows
.iter()
.flat_map(|row| row.iter().map(|field| Wire::from(field.as_str())))
.collect();
match connection
.exec_drop(&text, Params::Positional(values))
.await
{
Ok(()) => {
report.rows_written += batch.rows.len() as u64;
report.offset = batch.end;
let committed = confirm(connection).await;
if committed.is_ok() {
crate::i18n::report::load_batch_committed(report.rows_written, report.offset);
}
committed
}
Err(error) => {
let failure = map_driver_error(&error);
if !is_about_the_rows(&failure) {
return Err(failure);
}
if request.stop_on_refusal {
let rejection = Rejection {
row: batch.start,
reason: failure.message().to_owned(),
};
report.offset = batch.start;
let refusal = refused(&rejection);
report.rejected.push(rejection);
return Err(refusal);
}
note_lost_batch(batch, failure.message(), report);
Ok(())
}
}
}
fn is_about_the_rows(failure: &Failure) -> bool {
use crate::error::Category;
if failure.is_retryable() {
return false;
}
matches!(
failure.category(),
Category::Conflict | Category::Limit | Category::Server
)
}
fn note_lost_batch(batch: &Batch, reason: &str, report: &mut ImportReport) {
for &position in &batch.positions {
crate::i18n::report::load_row_rejected(position, reason);
report.rejected.push(Rejection {
row: position,
reason: reason.to_owned(),
});
}
report.offset = batch.end;
}
async fn confirm(connection: &mut Conn) -> Result<()> {
connection.query_drop("COMMIT").await.or_failure()
}
fn refused(rejection: &Rejection) -> Failure {
Failure::validation(
"datamove.row_refused",
format!("row {} was refused: {}", rejection.row, rejection.reason),
)
.with_detail("detail", rejection.row.to_string())
.with_hint("ask for refused rows to be skipped instead, to load the rest")
}
#[cfg(test)]
mod tests {
use super::*;
fn report() -> ImportReport {
ImportReport {
rows_read: 4,
rows_written: 2,
rejected: Vec::new(),
offset: 2,
columns: Vec::new(),
}
}
#[test]
fn the_statement_carries_placeholders_and_never_the_values() {
let text = statement("customers", &["id".to_owned(), "name".to_owned()], 2, 2)
.expect("legal names");
assert_eq!(
text,
"INSERT INTO `customers` (`id`, `name`) VALUES (?, ?), (?, ?)"
);
}
#[test]
fn a_file_that_names_no_columns_writes_into_the_table_as_declared() {
let text = statement("customers", &[], 1, 3).expect("legal names");
assert_eq!(text, "INSERT INTO `customers` VALUES (?, ?, ?)");
}
#[test]
fn a_name_this_product_will_not_write_is_refused_before_any_row_is_sent() {
assert!(statement("", &[], 1, 1).is_err(), "a nameless table");
assert!(
statement("customers", &["with\u{0}a control".to_owned()], 1, 1).is_err(),
"a column name with no quotable form"
);
}
#[test]
fn a_delimiter_inside_a_name_cannot_end_the_name_early() {
let text = statement("odd`name", &[], 1, 1).expect("quotable");
assert_eq!(text, "INSERT INTO `odd``name` VALUES (?)");
}
#[test]
fn stopping_at_the_first_refusal_ends_in_the_published_number() {
let failure = refused(&Rejection {
row: 7,
reason: "the line carries 1 field where the header names 2".to_owned(),
});
assert_eq!(failure.exit_number().code(), 3);
assert!(failure.message().contains('7'));
}
#[test]
fn only_a_refusal_a_row_can_cause_and_nobody_should_retry_is_skipped() {
let constraint =
Failure::conflict("conflict.constraint", "a key is held").with_retryable(false);
let too_long = Failure::limit("limit.value", "past a ceiling");
let stated = Failure::server("server.condition", "the value will not go in");
assert!(
is_about_the_rows(&constraint),
"a held key is the row's own"
);
assert!(is_about_the_rows(&too_long), "a ceiling is the row's own");
assert!(
is_about_the_rows(&stated),
"a stated condition may be the row"
);
let deadlock = Failure::conflict("conflict.deadlock", "the change hit a deadlock")
.with_retryable(true);
let no_room = Failure::server("server.too_many_connections", "no room right now")
.with_retryable(true);
let absent = Failure::absence("absence.unknown_object", "there is no such table");
assert!(
!is_about_the_rows(&deadlock),
"a deadlock is another session's doing, and the product says to repeat it"
);
assert!(
!is_about_the_rows(&no_room),
"having no free connection is not a property of any row"
);
assert!(
!is_about_the_rows(&absent),
"a table that is not there failed once, for the whole load"
);
}
#[test]
fn every_category_states_whether_a_row_can_cause_it() {
use crate::error::Category;
for category in Category::ALL {
let a_row_can_cause_it = match category {
Category::Conflict | Category::Limit | Category::Server => true,
Category::Usage
| Category::Validation
| Category::Config
| Category::Network
| Category::Authentication
| Category::Permission
| Category::Absence
| Category::Timeout
| Category::Interrupted
| Category::ChannelClosed => false,
};
let failure = Failure::new(category, "test.condition", "a stated condition")
.with_retryable(false);
assert_eq!(
is_about_the_rows(&failure),
a_row_can_cause_it,
"{category:?} must be decided by name, not by whichever side the list happens to hold"
);
}
}
#[test]
fn an_interrupted_load_says_where_to_resume_from() {
let failure = stopped(&report());
assert_eq!(failure.exit_number().code(), 130);
assert!(
failure.message().contains("offset 2"),
"{}",
failure.message()
);
assert!(failure.message().contains('2'));
}
}