use crate::cli::data::{DataCommand, ExportArgs, ImportArgs};
use crate::conn::Session;
use crate::datamove::export::ExportLock;
use crate::datamove::{export, import, ExportReport, ExportRequest, ImportReport, ImportRequest};
use crate::error::{Failure, Result};
use crate::output::Envelope;
use serde_json::{json, Value};
use tokio_util::sync::CancellationToken;
#[derive(Debug, Clone)]
pub struct DataSettings {
pub batch_size: usize,
pub rows_per_statement: usize,
pub volume_ceiling: Option<usize>,
pub stop_on_refusal: bool,
pub database: Option<String>,
pub export_lock: ExportLock,
}
impl DataSettings {
pub fn published() -> Result<Self> {
Ok(Self {
batch_size: builtin_count("import.batch_size")?,
rows_per_statement: builtin_count("data.rows_per_statement")?,
volume_ceiling: None,
stop_on_refusal: builtin_truth("data.stop_on_refusal")?,
database: None,
export_lock: builtin_lock("data.export_lock")?,
})
}
}
fn registry_defect(name: &str, what: &str) -> Failure {
Failure::config("config.registry_defect", format!("`{name}` {what}"))
.with_detail("detail", name)
.with_hint("this is a defect in the built-in setting registry, not in the call")
}
fn builtin_count(name: &str) -> Result<usize> {
match builtin(name)? {
crate::config::keys::Value::Integer(value) => {
Ok(usize::try_from(value).unwrap_or(usize::MAX))
}
other => Err(registry_defect(
name,
&format!("is not a number in the registry: {other}"),
)),
}
}
fn builtin_truth(name: &str) -> Result<bool> {
match builtin(name)? {
crate::config::keys::Value::Boolean(value) => Ok(value),
other => Err(registry_defect(
name,
&format!("is not a truth value in the registry: {other}"),
)),
}
}
fn builtin_lock(name: &str) -> Result<ExportLock> {
match builtin(name)? {
crate::config::keys::Value::Enumeration(word) => ExportLock::parse(word)
.map_err(|_| registry_defect(name, "publishes a word outside its own domain")),
other => Err(registry_defect(
name,
&format!("is not a word in the registry: {other}"),
)),
}
}
fn builtin(name: &str) -> Result<crate::config::keys::Value> {
crate::config::keys::spec(name)
.ok_or_else(|| registry_defect(name, "is not a published setting"))?
.default_value()
.map_err(|_| registry_defect(name, "carries a built-in value outside its own domain"))?
.ok_or_else(|| registry_defect(name, "carries no built-in value"))
}
pub async fn execute(
command: &DataCommand,
session: &mut Session,
settings: &DataSettings,
token: &CancellationToken,
) -> Result<Envelope> {
match command {
DataCommand::Export(args) => {
let request = export_request(args, settings)?;
let report = export::run(session.connection(), &request, token).await?;
Ok(export_envelope(&report))
}
DataCommand::Import(args) => {
let request = import_request(args, settings)?;
let report = import::run(session.connection(), &request, token).await?;
Ok(import_envelope(&report))
}
}
}
fn tables_of(named: &[String], in_force: Option<String>) -> Result<(Option<String>, Vec<String>)> {
let mut carried: Option<String> = None;
let mut bare = Vec::with_capacity(named.len());
for name in named {
let split = crate::introspect::names::split(name);
if let Some(database) = split.database {
match &carried {
Some(first) if !first.eq_ignore_ascii_case(&database) => {
return Err(Failure::usage(
"data.databases_disagree",
format!("`{first}` and `{database}`"),
)
.with_detail("detail", format!("`{first}` and `{database}`"))
.with_hint("run one copy for each database"));
}
Some(_) => {}
None => carried = Some(database),
}
}
bare.push(split.name);
}
Ok((carried.or(in_force), bare))
}
fn export_request(args: &ExportArgs, settings: &DataSettings) -> Result<ExportRequest> {
let output = args.to.clone().ok_or_else(|| {
Failure::usage(
"data.output_missing",
"the copy needs a destination to write to",
)
.with_hint("name one with --to")
})?;
let (database, tables) = tables_of(&args.table, settings.database.clone())?;
Ok(ExportRequest {
database,
tables,
structure_only: args.structure_only,
output,
ceiling: args.max_bytes.or(settings.volume_ceiling),
rows_per_statement: settings.rows_per_statement,
lock: settings.export_lock,
})
}
fn import_request(args: &ImportArgs, settings: &DataSettings) -> Result<ImportRequest> {
const FORM: &str = "data import --file clientes.csv --table clientes";
let file = args.file.clone().ok_or_else(|| {
Failure::usage("data.file_missing", "the load needs a file to read")
.with_hint(format!("name one with --file, as in `{FORM}`"))
})?;
let table = args.table.clone().ok_or_else(|| {
Failure::usage("data.table_missing", "the load needs a table to write into")
.with_hint(format!("name one with --table, as in `{FORM}`"))
})?;
let separator = match args.separator {
None => b',',
Some(character) => {
let mut buffer = [0_u8; 4];
let encoded = character.encode_utf8(&mut buffer);
match encoded.len() {
1 => encoded.as_bytes()[0],
_ => {
return Err(crate::error::Failure::usage(
"datamove.separator_too_wide",
format!(
"`{character}` is wider than one byte, so it cannot separate fields"
),
)
.with_detail("detail", character.to_string())
.with_hint("use a separator that is one byte, such as a comma or a tab"))
}
}
}
};
let split = crate::introspect::names::split(&table);
Ok(ImportRequest {
file,
table: split.name.clone(),
database: split.database_or(settings.database.clone()),
batch_size: args.batch_size.unwrap_or(settings.batch_size),
resume_from: args.resume_from.unwrap_or(0),
separator,
header: args.header,
stop_on_refusal: args
.on_error
.map_or(settings.stop_on_refusal, crate::cli::data::OnError::stops),
})
}
#[must_use]
pub fn export_envelope(report: &ExportReport) -> Envelope {
let envelope = Envelope::new(
"data export",
json!({
"path": report.path.display().to_string(),
"objects": report.objects,
"rows": report.rows,
"bytes": report.bytes,
"content_checksum": report.content_checksum,
"consistent": report.consistent,
}),
);
report
.gaps
.iter()
.fold(envelope, |carried, gap| carried.with_gap(gap.clone()))
}
#[must_use]
pub fn import_envelope(report: &ImportReport) -> Envelope {
let rows: Vec<Value> = report
.rejected
.iter()
.map(|rejection| json!({"row": rejection.row, "reason": rejection.reason}))
.collect();
Envelope::new(
"data import",
json!({
"rows_read": report.rows_read,
"rows_written": report.rows_written,
"rows_rejected": rows.len(),
"offset": report.offset,
"columns": report.columns,
"rows": rows,
}),
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::output::envelope::Gap;
fn defaults() -> DataSettings {
DataSettings::published().expect("the registry publishes these settings")
}
fn export_args() -> ExportArgs {
ExportArgs {
max_bytes: None,
to: Some(std::path::PathBuf::from("copy.sql")),
table: vec!["orders".to_owned()],
structure_only: false,
}
}
fn import_args() -> ImportArgs {
ImportArgs {
on_error: None,
file: Some(std::path::PathBuf::from("rows.csv")),
table: Some("customers".to_owned()),
batch_size: None,
resume_from: None,
separator: None,
header: true,
}
}
#[test]
fn the_stored_batch_size_applies_when_no_mark_was_given() {
let settings = defaults();
let request = import_request(&import_args(), &settings).expect("legal");
assert_eq!(request.batch_size, settings.batch_size);
assert_eq!(request.separator, b',');
assert_eq!(request.resume_from, 0);
}
#[test]
fn a_mark_outranks_the_stored_batch_size() {
let mut args = import_args();
args.batch_size = Some(50);
args.resume_from = Some(800_000);
let request = import_request(&args, &defaults()).expect("legal");
assert_eq!(request.batch_size, 50);
assert_eq!(request.resume_from, 800_000);
}
#[test]
fn a_separator_of_one_byte_is_taken_as_given() {
let mut args = import_args();
args.separator = Some('\t');
assert_eq!(
import_request(&args, &defaults()).expect("legal").separator,
b'\t'
);
}
#[test]
fn a_load_takes_the_qualified_name_a_map_wrote() {
let mut args = import_args();
args.table = Some("shop.customers".to_owned());
let request = import_request(&args, &defaults()).expect("legal");
assert_eq!(request.table, "customers");
assert_eq!(request.database.as_deref(), Some("shop"));
}
#[test]
fn the_database_a_loaded_name_carries_wins_over_the_one_in_force() {
let settings = DataSettings {
database: Some("other".to_owned()),
..defaults()
};
let mut args = import_args();
args.table = Some("shop.customers".to_owned());
assert_eq!(
import_request(&args, &settings)
.expect("legal")
.database
.as_deref(),
Some("shop")
);
assert_eq!(
import_request(&import_args(), &settings)
.expect("legal")
.database
.as_deref(),
Some("other")
);
}
#[test]
fn a_copy_of_tables_from_two_databases_is_refused_where_it_is_asked_for() {
let mut args = export_args();
args.table = vec!["shop.orders".to_owned(), "other.clients".to_owned()];
let failure = export_request(&args, &defaults()).expect_err("must refuse");
assert_eq!(failure.code(), "data.databases_disagree");
assert_eq!(failure.exit_number().code(), 2);
let mut agreeing = export_args();
agreeing.table = vec!["shop.orders".to_owned(), "SHOP.clients".to_owned()];
let request = export_request(&agreeing, &defaults()).expect("one database");
assert_eq!(request.database.as_deref(), Some("shop"));
assert_eq!(
request.tables,
vec!["orders".to_owned(), "clients".to_owned()]
);
}
#[test]
fn a_separator_wider_than_a_byte_is_refused_before_the_file_is_opened() {
let mut args = import_args();
args.separator = Some('—');
let failure = import_request(&args, &defaults()).expect_err("must refuse");
assert_eq!(failure.code(), "datamove.separator_too_wide");
assert_eq!(failure.exit_number().code(), 2);
}
#[test]
fn the_marks_of_a_copy_reach_the_request_unchanged() {
let settings = DataSettings {
database: Some("shop".to_owned()),
..defaults()
};
let request = export_request(&export_args(), &settings).expect("a destination is set");
assert_eq!(
request.database.as_deref(),
Some("shop"),
"the one global --database reaches the copy"
);
assert_eq!(request.tables, vec!["orders".to_owned()]);
assert!(!request.structure_only);
}
#[test]
fn a_copy_without_a_destination_is_refused_with_a_code() {
let mut args = export_args();
args.to = None;
let failure = export_request(&args, &defaults()).expect_err("must refuse");
assert_eq!(failure.code(), "data.output_missing");
assert_eq!(failure.exit_number().code(), 2);
assert!(failure.hint().is_some());
}
#[test]
fn a_copy_that_is_not_a_snapshot_declares_the_shortfall() {
let report = ExportReport {
path: std::path::PathBuf::from("/tmp/copy.sql"),
objects: 2,
rows: 10,
bytes: 512,
content_checksum: "sha256:abc".to_owned(),
consistent: false,
gaps: vec![Gap::new(
"sessions",
"no_consistent_snapshot",
"the engine keeps no versions",
)],
};
let parsed = serde_json::to_value(export_envelope(&report)).expect("serialises");
assert_eq!(parsed["command"], "data export");
assert_eq!(parsed["data"]["consistent"], false);
assert_eq!(parsed["gaps"][0]["cause"], "no_consistent_snapshot");
}
#[test]
fn a_whole_copy_carries_no_shortfall_at_all() {
let report = ExportReport {
path: std::path::PathBuf::from("/tmp/copy.sql"),
objects: 1,
rows: 3,
bytes: 64,
content_checksum: "sha256:abc".to_owned(),
consistent: true,
gaps: Vec::new(),
};
let line = serde_json::to_string(&export_envelope(&report)).expect("serialises");
assert!(!line.contains("gaps"));
assert!(line.contains("content_checksum"));
}
#[test]
fn a_load_reports_the_reason_of_every_refusal_and_where_to_resume() {
let report = ImportReport {
rows_read: 5,
rows_written: 3,
rejected: vec![
crate::datamove::Rejection {
row: 1,
reason: "the line carries 1 field where the header names 2".to_owned(),
},
crate::datamove::Rejection {
row: 4,
reason: "the column will not take this value".to_owned(),
},
],
offset: 5,
columns: vec!["id".to_owned(), "name".to_owned()],
};
let parsed = serde_json::to_value(import_envelope(&report)).expect("serialises");
assert_eq!(parsed["command"], "data import");
assert_eq!(parsed["data"]["rows_written"], 3);
assert_eq!(parsed["data"]["rows_rejected"], 2);
assert_eq!(parsed["data"]["offset"], 5);
assert_eq!(parsed["data"]["rows"][0]["row"], 1);
assert!(parsed["data"]["rows"][1]["reason"]
.as_str()
.is_some_and(|reason| !reason.is_empty()));
}
#[test]
fn the_refusals_sit_under_the_published_payload_key() {
let report = ImportReport {
rows_read: 1,
rows_written: 1,
rejected: Vec::new(),
offset: 1,
columns: Vec::new(),
};
assert_eq!(import_envelope(&report).payload_array_key(), Some("rows"));
}
}