use super::AtomicWriter;
use crate::conn::OrFailure;
use crate::error::{Failure, Result};
use crate::output::checksum::Signature;
use crate::output::envelope::{Gap, Row};
use crate::sql::ident;
use mysql_async::prelude::Queryable;
use mysql_async::{Conn, Value as Wire};
use tokio_util::sync::CancellationToken;
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum ExportLock {
#[default]
None,
Tables,
}
impl ExportLock {
pub fn parse(word: &str) -> Result<Self> {
match word {
"none" => Ok(Self::None),
"tables" => Ok(Self::Tables),
other => Err(Failure::config(
"config.unknown_export_lock",
format!("`{other}` is not something a copy can hold still"),
)
.with_detail("detail", other)
.with_hint("set data.export_lock to none or to tables")),
}
}
#[must_use]
pub const fn holds_the_tables(self) -> bool {
matches!(self, Self::Tables)
}
}
pub fn hold_statement(tables: &[String]) -> Result<Option<String>> {
if tables.is_empty() {
return Ok(None);
}
let mut named = Vec::with_capacity(tables.len());
for table in tables {
named.push(format!("{} READ", ident::quote(table)?));
}
Ok(Some(format!("LOCK TABLES {}", named.join(", "))))
}
#[derive(Debug, Clone)]
pub struct ExportRequest {
pub database: Option<String>,
pub tables: Vec<String>,
pub structure_only: bool,
pub output: std::path::PathBuf,
pub ceiling: Option<usize>,
pub rows_per_statement: usize,
pub lock: ExportLock,
}
#[derive(Debug, Clone)]
pub struct ExportReport {
pub path: std::path::PathBuf,
pub objects: usize,
pub rows: u64,
pub bytes: usize,
pub content_checksum: String,
pub consistent: bool,
pub gaps: Vec<Gap>,
}
const SNAPSHOT_ENGINES: [&str; 2] = ["innodb", "xtradb"];
#[must_use]
pub fn keeps_a_snapshot(engine: &str) -> bool {
SNAPSHOT_ENGINES.contains(&engine.trim().to_ascii_lowercase().as_str())
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct TableFacts {
name: String,
engine: String,
}
pub async fn run(
connection: &mut Conn,
request: &ExportRequest,
token: &CancellationToken,
) -> Result<ExportReport> {
let tables = list_tables(connection, request).await?;
crate::i18n::report::copy_started(tables.len(), &request.output.display().to_string());
let mut writer = AtomicWriter::create(&request.output, request.ceiling)?;
writer.write(&header())?;
let held = request.lock.holds_the_tables();
let outside: Vec<&TableFacts> = if held {
Vec::new()
} else {
tables
.iter()
.filter(|table| !keeps_a_snapshot(&table.engine))
.collect()
};
for table in &outside {
crate::i18n::report::copy_snapshot_unavailable(&table.name);
}
let gaps = outside.iter().map(|table| snapshot_gap(table)).collect();
let consistent = outside.is_empty();
if held {
hold_tables(connection, &tables).await?;
}
begin_snapshot(connection).await?;
let outcome = copy_all(connection, request, &tables, &mut writer, token).await;
let finished = finish_snapshot(connection).await;
let released = if held {
release_tables(connection).await
} else {
Ok(())
};
let rows = outcome?;
finished?;
released?;
writer.write("COMMIT;\n")?;
let written = writer.commit()?;
crate::i18n::report::copy_finished(rows.count, written.bytes as u64);
Ok(ExportReport {
path: written.path,
objects: tables.len(),
rows: rows.count,
bytes: written.bytes,
content_checksum: rows.checksum,
consistent,
gaps,
})
}
fn snapshot_gap(table: &TableFacts) -> Gap {
Gap::new(
table.name.clone(),
"no_consistent_snapshot",
format!(
"`{}` is on the `{}` engine, which keeps no versions to read from, so this table is not part of the agreement between the others",
table.name, table.engine
),
)
}
struct RowTotals {
count: u64,
checksum: String,
}
async fn copy_all(
connection: &mut Conn,
request: &ExportRequest,
tables: &[TableFacts],
writer: &mut AtomicWriter,
token: &CancellationToken,
) -> Result<RowTotals> {
let mut signature = Signature::new();
let mut count = 0;
for table in tables {
if token.is_cancelled() {
return Err(Failure::interrupted(
"interrupted.during_copy",
"the invocation was stopped while the copy was being written",
));
}
crate::i18n::report::copy_table(&table.name);
writer.write(&format!("\n-- table {}\n", table.name))?;
writer.write(&structure_of(connection, &table.name).await?)?;
if !request.structure_only {
count += copy_rows(
connection,
request,
&table.name,
writer,
&mut signature,
token,
)
.await?;
}
}
Ok(RowTotals {
count,
checksum: signature.finish(),
})
}
pub(super) fn header() -> String {
concat!(
"-- logical copy produced by ",
env!("CARGO_PKG_NAME"),
"\n",
"SET @saved_sql_mode = @@SQL_MODE;\n",
"SET SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO,NO_BACKSLASH_ESCAPES';\n",
"SET @saved_foreign_key_checks = @@FOREIGN_KEY_CHECKS;\n",
"SET FOREIGN_KEY_CHECKS = 0;\n",
"START TRANSACTION;\n",
)
.to_owned()
}
async fn begin_snapshot(connection: &mut Conn) -> Result<()> {
connection
.query_drop("START TRANSACTION WITH CONSISTENT SNAPSHOT")
.await
.or_failure()
}
async fn hold_tables(connection: &mut Conn, tables: &[TableFacts]) -> Result<()> {
let names: Vec<String> = tables.iter().map(|table| table.name.clone()).collect();
let Some(statement) = hold_statement(&names)? else {
return Ok(());
};
connection.query_drop(statement).await.or_failure()
}
async fn release_tables(connection: &mut Conn) -> Result<()> {
connection.query_drop("UNLOCK TABLES").await.or_failure()
}
async fn finish_snapshot(connection: &mut Conn) -> Result<()> {
connection.query_drop("COMMIT").await.or_failure()
}
async fn list_tables(connection: &mut Conn, request: &ExportRequest) -> Result<Vec<TableFacts>> {
let statement = "SELECT TABLE_NAME, IFNULL(ENGINE, '') \
FROM information_schema.TABLES \
WHERE TABLE_SCHEMA = IFNULL(?, DATABASE()) AND TABLE_TYPE = 'BASE TABLE' \
ORDER BY TABLE_NAME";
let found: Vec<(String, String)> = connection
.exec(statement, (request.database.clone(),))
.await
.or_failure()?;
let mut facts: Vec<TableFacts> = found
.into_iter()
.map(|(name, engine)| TableFacts { name, engine })
.collect();
if !request.tables.is_empty() {
facts.retain(|table| request.tables.contains(&table.name));
for wanted in &request.tables {
if !facts.iter().any(|table| table.name == *wanted) {
return Err(Failure::absence(
"datamove.table_absent",
format!("`{wanted}` is not a table of this database"),
)
.with_detail("detail", wanted.clone())
.with_hint("run `schema tables` to see what is there"));
}
}
}
Ok(facts)
}
async fn structure_of(connection: &mut Conn, table: &str) -> Result<String> {
let quoted = ident::quote(table)?;
let created: Option<(String, String)> = connection
.query_first(format!("SHOW CREATE TABLE {quoted}"))
.await
.or_failure()?;
let (_, definition) = created.ok_or_else(|| {
Failure::absence(
"datamove.structure_absent",
format!("the server described no structure for `{table}`"),
)
.with_detail("detail", table)
})?;
Ok(format!("DROP TABLE IF EXISTS {quoted};\n{definition};\n"))
}
async fn copy_rows(
connection: &mut Conn,
request: &ExportRequest,
table: &str,
writer: &mut AtomicWriter,
signature: &mut Signature,
token: &CancellationToken,
) -> Result<u64> {
let quoted = ident::quote(table)?;
let mut result = connection
.query_iter(format!("SELECT * FROM {quoted}"))
.await
.or_failure()?;
let columns = result.columns_ref().to_vec();
let mut batcher = Batcher::new(writer, quoted, request.rows_per_statement);
while let Some(row) = result.next().await.or_failure()? {
if token.is_cancelled() {
return Err(Failure::interrupted(
"interrupted.during_copy",
format!("the copy of `{table}` was stopped while its rows were still arriving"),
));
}
batcher.push(&row, &columns, signature);
}
batcher.finish()
}
struct Batcher<'a> {
writer: &'a mut AtomicWriter,
quoted: String,
per_statement: usize,
pending: Vec<String>,
count: u64,
held: Option<Failure>,
}
impl<'a> Batcher<'a> {
fn new(writer: &'a mut AtomicWriter, quoted: String, per_statement: usize) -> Self {
let per_statement = per_statement.max(1);
Self {
writer,
quoted,
per_statement,
pending: Vec::with_capacity(per_statement),
count: 0,
held: None,
}
}
fn push(
&mut self,
row: &mysql_async::Row,
columns: &[mysql_async::Column],
signature: &mut Signature,
) {
if self.held.is_some() {
return;
}
if let Err(failure) = self.take(row, columns, signature) {
self.held = Some(failure);
}
}
fn take(
&mut self,
row: &mysql_async::Row,
columns: &[mysql_async::Column],
signature: &mut Signature,
) -> Result<()> {
self.pending.push(render_row(row, columns, signature)?);
self.count += 1;
if self.pending.len() >= self.per_statement {
self.flush()?;
}
Ok(())
}
fn flush(&mut self) -> Result<()> {
if self.pending.is_empty() {
return Ok(());
}
let statement = format!(
"INSERT INTO {} VALUES {};\n",
self.quoted,
self.pending.join(", ")
);
self.pending.clear();
self.writer.write(&statement)
}
fn finish(mut self) -> Result<u64> {
if let Some(failure) = self.held.take() {
return Err(failure);
}
self.flush()?;
Ok(self.count)
}
}
fn render_row(
row: &mysql_async::Row,
columns: &[mysql_async::Column],
signature: &mut Signature,
) -> Result<String> {
const ABSENT: Wire = Wire::NULL;
let mut rendered = Row::new();
let mut literals = Vec::with_capacity(columns.len());
for (index, column) in columns.iter().enumerate() {
let wire = row.as_ref(index).unwrap_or(&ABSENT);
literals.push(super::literal::literal(wire, column));
rendered.insert(
column.name_str().into_owned(),
signature_value(wire, column)?,
);
}
signature.absorb(&rendered);
Ok(format!("({})", literals.join(", ")))
}
fn signature_value(wire: &Wire, column: &mysql_async::Column) -> Result<serde_json::Value> {
crate::types::convert(wire, column, &crate::types::Conversion::stable())
}
#[cfg(test)]
mod tests {
use super::*;
fn published_with(binary: &str) -> crate::types::Conversion {
let scratch = tempfile::tempdir().expect("a scratch directory");
let file =
crate::config::paths::config_file(Some(scratch.path())).expect("a settings location");
crate::config::store::store(&file, "output.binary", binary).expect("legal");
let config = crate::config::Config::load(Some(scratch.path())).expect("the location loads");
crate::settings::conversion(&config).expect("resolves")
}
fn byte_column() -> mysql_async::Column {
mysql_async::Column::new(mysql_async::consts::ColumnType::MYSQL_TYPE_BLOB)
.with_character_set(crate::types::BINARY_CHARSET)
}
#[test]
fn the_signature_of_identical_rows_never_moves_with_a_display_setting() {
let column = byte_column();
let bytes = vec![7u8; 64];
let described = crate::types::convert(
&Wire::Bytes(bytes.clone()),
&column,
&published_with("summary"),
)
.expect("converts");
let whole = crate::types::convert(
&Wire::Bytes(bytes.clone()),
&column,
&published_with("full"),
)
.expect("converts");
assert_ne!(
described, whole,
"the setting has to move the shaping, or this test proves nothing"
);
let signed_once = signature_value(&Wire::Bytes(bytes.clone()), &column).expect("converts");
let signed_again = signature_value(&Wire::Bytes(bytes), &column).expect("converts");
assert_eq!(signed_once, signed_again);
assert_ne!(
signed_once, whole,
"the signature must not follow the setting into the whole form"
);
let digest = |value: &serde_json::Value| {
let mut row = Row::new();
row.insert("payload".to_owned(), value.clone());
let mut signature = Signature::new();
signature.absorb(&row);
signature.finish()
};
assert_eq!(
digest(&signed_once),
digest(&signed_again),
"identical rows must sign identically whatever the setting says"
);
}
#[test]
fn a_signature_moves_when_the_content_it_signs_moves() {
use mysql_async::consts::ColumnType;
let digest = |wire: &Wire, column: &mysql_async::Column| {
let mut row = Row::new();
row.insert(
"payload".to_owned(),
signature_value(wire, column).expect("converts"),
);
let mut signature = Signature::new();
signature.absorb(&row);
signature.finish()
};
let one = Wire::Bytes(vec![0x11u8; 64]);
let other = Wire::Bytes(vec![0x22u8; 64]);
for column in [
byte_column(),
mysql_async::Column::new(ColumnType::MYSQL_TYPE_VECTOR),
mysql_async::Column::new(ColumnType::MYSQL_TYPE_GEOMETRY),
] {
assert_ne!(
digest(&one, &column),
digest(&other, &column),
"two different values signed the same, so the signature is blind to {:?}",
column.column_type()
);
}
}
#[test]
fn only_an_engine_that_keeps_versions_joins_the_agreement() {
assert!(keeps_a_snapshot("InnoDB"));
assert!(keeps_a_snapshot("innodb"));
assert!(keeps_a_snapshot(" XtraDB "));
assert!(!keeps_a_snapshot("MyISAM"));
assert!(!keeps_a_snapshot("MEMORY"));
assert!(!keeps_a_snapshot("ARCHIVE"));
assert!(!keeps_a_snapshot(""));
}
#[test]
fn a_table_outside_the_agreement_is_declared_rather_than_hidden() {
let gap = snapshot_gap(&TableFacts {
name: "sessions".to_owned(),
engine: "MEMORY".to_owned(),
});
assert_eq!(gap.object, "sessions");
assert_eq!(gap.cause, "no_consistent_snapshot");
assert!(gap.detail.contains("MEMORY"));
}
#[test]
fn the_header_puts_the_reader_in_a_known_state() {
let opened = header();
assert!(opened.contains("SET SQL_MODE"));
assert!(opened.contains("FOREIGN_KEY_CHECKS = 0"));
assert!(opened.contains("START TRANSACTION;"));
}
}