use crate::core::cross_db_conversion::{
ConversionReport, CrossDbConverter, FieldLink, FieldLinker, LinkKind, TargetDbKind,
TypeMappingTable, ValueTransformer,
};
use crate::databases::{DatabaseCompatibility, DatabaseVersion};
use crate::models::{Field, TableSchema};
use anyhow::Result;
use chrono::Utc;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum MigrationDirection {
SameDbCrossVersion,
CrossDbSameVersion,
CrossDbCrossVersion,
SameDbSameVersion,
}
impl MigrationDirection {
pub fn name(&self) -> &'static str {
match self {
MigrationDirection::SameDbCrossVersion => "同库跨版本",
MigrationDirection::CrossDbSameVersion => "异构同版本",
MigrationDirection::CrossDbCrossVersion => "异构跨版本",
MigrationDirection::SameDbSameVersion => "同库同版本",
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MigrationConfig {
pub source_db: TargetDbKind,
pub target_db: TargetDbKind,
pub source_version: Option<DatabaseVersion>,
pub target_version: Option<DatabaseVersion>,
pub auto_field_link: bool,
pub manual_field_map: HashMap<String, String>,
pub batch_size: usize,
pub enable_version_upgrade: bool,
pub pre_check: bool,
pub default_source_version: Option<DatabaseVersion>,
pub default_target_version: Option<DatabaseVersion>,
}
impl Default for MigrationConfig {
fn default() -> Self {
Self {
source_db: TargetDbKind::MySQL,
target_db: TargetDbKind::PostgreSQL,
source_version: None,
target_version: None,
auto_field_link: true,
manual_field_map: HashMap::new(),
batch_size: 1000,
enable_version_upgrade: true,
pre_check: true,
default_source_version: None,
default_target_version: None,
}
}
}
impl MigrationConfig {
pub fn effective_source_version(&self) -> DatabaseVersion {
self.source_version
.clone()
.or_else(|| self.default_source_version.clone())
.unwrap_or_else(|| default_version_for(self.source_db))
}
pub fn effective_target_version(&self) -> DatabaseVersion {
self.target_version
.clone()
.or_else(|| self.default_target_version.clone())
.unwrap_or_else(|| default_version_for(self.target_db))
}
pub fn direction(&self) -> MigrationDirection {
if self.source_db == self.target_db {
if self.effective_source_version().major == self.effective_target_version().major
&& self.effective_source_version().minor == self.effective_target_version().minor
{
MigrationDirection::SameDbSameVersion
} else {
MigrationDirection::SameDbCrossVersion
}
} else if self.effective_source_version().major == self.effective_target_version().major
&& self.effective_source_version().minor == self.effective_target_version().minor
{
MigrationDirection::CrossDbSameVersion
} else {
MigrationDirection::CrossDbCrossVersion
}
}
}
fn default_version_for(db: TargetDbKind) -> DatabaseVersion {
match db {
TargetDbKind::MySQL | TargetDbKind::MariaDB | TargetDbKind::TiDB => {
DatabaseVersion::new(8, 0, 32)
}
TargetDbKind::PostgreSQL => DatabaseVersion::new(16, 2, 0),
TargetDbKind::SQLite => DatabaseVersion::new(3, 45, 0),
TargetDbKind::Oracle => DatabaseVersion::new(21, 0, 0),
TargetDbKind::MSSQL => DatabaseVersion::new(16, 0, 0),
_ => DatabaseVersion::new(1, 0, 0),
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FieldMigration {
pub source_field: String,
pub target_field: String,
pub source_type: String,
pub target_type: String,
pub transform: String,
pub lossy: bool,
pub warnings: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TableMigrationResult {
pub table_name: String,
pub direction: MigrationDirection,
pub source_db: TargetDbKind,
pub target_db: TargetDbKind,
pub source_version: String,
pub target_version: String,
pub fields_total: usize,
pub fields_mapped: usize,
pub fields_unmapped: usize,
pub lossy_conversions: usize,
pub warnings: Vec<String>,
pub field_migrations: Vec<FieldMigration>,
pub ddl: String,
pub started_at: String,
pub finished_at: String,
pub elapsed_ms: u64,
}
impl TableMigrationResult {
pub fn success_rate(&self) -> f64 {
if self.fields_total == 0 {
0.0
} else {
self.fields_mapped as f64 / self.fields_total as f64
}
}
}
pub struct DataMigrator {
converter: CrossDbConverter,
}
impl DataMigrator {
pub fn new() -> Self {
Self {
converter: CrossDbConverter::new(),
}
}
pub fn add_type_rule(
&mut self,
source: TargetDbKind,
target: TargetDbKind,
source_type: &str,
target_type: &str,
lossy: bool,
) {
self.converter.add_rule(crate::core::cross_db_conversion::TypeMappingRule {
source_db: source,
source_type_pattern: source_type.to_string(),
target_db: target,
target_type: target_type.to_string(),
note: "用户自定义规则".to_string(),
lossy,
});
}
pub fn migrate_table(
&self,
source_table: &TableSchema,
config: &MigrationConfig,
) -> Result<TableMigrationResult> {
let started_at = Utc::now();
let src_v = config.effective_source_version();
let tgt_v = config.effective_target_version();
let direction = config.direction();
let upgraded_source = if config.enable_version_upgrade && direction != MigrationDirection::SameDbSameVersion
{
apply_version_upgrade(source_table, config.source_db, &src_v, &tgt_v)
} else {
source_table.clone()
};
let linker = if config.manual_field_map.is_empty() {
FieldLinker::new()
} else {
FieldLinker::new().with_manual(config.manual_field_map.clone())
};
let field_migrations = build_field_migrations(
&upgraded_source.fields,
&linker,
config.source_db,
config.target_db,
&src_v,
&tgt_v,
&self.converter,
)?;
let mapped = field_migrations
.iter()
.filter(|m| !m.target_field.is_empty())
.count();
let unmapped = field_migrations.len() - mapped;
let lossy_count = field_migrations.iter().filter(|m| m.lossy).count();
let warnings: Vec<String> = field_migrations
.iter()
.flat_map(|m| m.warnings.iter().cloned())
.collect();
let ddl = self.generate_ddl(&upgraded_source, config, &field_migrations)?;
let finished_at = Utc::now();
let elapsed_ms = (finished_at - started_at).num_milliseconds() as u64;
Ok(TableMigrationResult {
table_name: source_table.name.clone(),
direction,
source_db: config.source_db,
target_db: config.target_db,
source_version: src_v.to_string(),
target_version: tgt_v.to_string(),
fields_total: field_migrations.len(),
fields_mapped: mapped,
fields_unmapped: unmapped,
lossy_conversions: lossy_count,
warnings,
field_migrations,
ddl,
started_at: started_at.to_rfc3339(),
finished_at: finished_at.to_rfc3339(),
elapsed_ms,
})
}
fn generate_ddl(
&self,
source: &TableSchema,
config: &MigrationConfig,
migrations: &[FieldMigration],
) -> Result<String> {
let mut cols = Vec::new();
for m in migrations {
if m.target_field.is_empty() {
continue;
}
let quote = |s: &str| -> String {
match config.target_db {
TargetDbKind::MySQL | TargetDbKind::MariaDB | TargetDbKind::TiDB => {
format!("`{}`", s.replace('`', "``"))
}
_ => format!("\"{}\"", s.replace('"', "\"\"")),
}
};
cols.push(format!(" {} {}", quote(&m.target_field), m.target_type));
}
Ok(format!(
"CREATE TABLE {} (\n{}\n)",
quote(&source.name,),
cols.join(",\n")
).replace("\"", "\""))
}
}
impl Default for DataMigrator {
fn default() -> Self {
Self::new()
}
}
fn quote(name: &str) -> String {
format!("\"{}\"", name)
}
fn apply_version_upgrade(
table: &TableSchema,
db: TargetDbKind,
src_v: &DatabaseVersion,
tgt_v: &DatabaseVersion,
) -> TableSchema {
let mut upgraded = table.clone();
if !src_v.is_lt(tgt_v) {
return upgraded;
}
for f in &mut upgraded.fields {
upgrade_field_type(f, db, src_v, tgt_v);
}
upgraded
}
fn upgrade_field_type(
f: &mut Field,
db: TargetDbKind,
src_v: &DatabaseVersion,
tgt_v: &DatabaseVersion,
) {
let upper = f.data_type.to_ascii_uppercase();
match db {
TargetDbKind::MySQL | TargetDbKind::MariaDB | TargetDbKind::TiDB => {
if src_v.major < 8 && tgt_v.major >= 8 {
if upper == "DATETIME" {
f.data_type = "DATETIME(6)".to_string();
}
if upper == "TINYINT(1)" {
f.data_type = "BOOLEAN".to_string();
}
}
}
TargetDbKind::PostgreSQL => {
if src_v.major < 10 && tgt_v.major >= 10 {
if upper.contains("SERIAL") {
let base = upper.split_whitespace().next().unwrap_or("INT");
f.data_type = format!("{} GENERATED BY DEFAULT AS IDENTITY", base);
}
}
if src_v.major < 14 && tgt_v.major >= 14 {
}
}
TargetDbKind::SQLite => {
if (src_v.major, src_v.minor) < (3, 38) && (tgt_v.major, tgt_v.minor) >= (3, 38) {
if upper == "JSON" {
f.data_type = "JSONB".to_string();
}
}
}
_ => {}
}
}
fn build_field_migrations(
source_fields: &[Field],
linker: &FieldLinker,
source_db: TargetDbKind,
target_db: TargetDbKind,
src_v: &DatabaseVersion,
tgt_v: &DatabaseVersion,
converter: &CrossDbConverter,
) -> Result<Vec<FieldMigration>> {
let target_field_names: Vec<String> = source_fields.iter().map(|f| f.name.clone()).collect();
let target_fields: Vec<Field> = source_fields
.iter()
.map(|f| Field {
name: f.name.clone(),
data_type: f.data_type.clone(),
length: f.length,
nullable: f.nullable,
default_value: f.default_value.clone(),
primary_key: f.primary_key,
auto_increment: f.auto_increment,
})
.collect();
let links: Vec<FieldLink> = linker.link(source_fields, &target_fields);
let table = &converter.type_table;
let mut out = Vec::new();
for (i, sf) in source_fields.iter().enumerate() {
let link = &links[i];
if link.target_field.is_empty() {
out.push(FieldMigration {
source_field: sf.name.clone(),
target_field: String::new(),
source_type: sf.data_type.clone(),
target_type: String::new(),
transform: "skipped".to_string(),
lossy: false,
warnings: vec!["字段未匹配,未包含在目标 DDL 中".to_string()],
});
continue;
}
let rule = table.lookup_with_source(&sf.data_type, source_db, target_db);
let (target_type, lossy) = match rule {
Some(r) => (r.target_type.clone(), r.lossy),
None => {
(sf.data_type.clone(), false)
}
};
let final_type = if src_v.major != tgt_v.major || src_v.minor != tgt_v.minor {
adjust_type_by_version(&target_type, target_db, tgt_v)
} else {
target_type.clone()
};
let mut warnings = Vec::new();
if lossy {
warnings.push(format!(
"{} → {} 可能损失精度",
sf.data_type, final_type
));
}
if final_type != target_type {
warnings.push(format!("目标版本 {} 启用增强: {}", tgt_v, final_type));
}
if (source_db, target_db) == (TargetDbKind::MySQL, TargetDbKind::PostgreSQL) {
if sf.data_type.eq_ignore_ascii_case("TIMESTAMP") {
warnings.push("MySQL TIMESTAMP → PostgreSQL TIMESTAMPTZ(保留时区)".to_string());
}
}
if (source_db, target_db) == (TargetDbKind::PostgreSQL, TargetDbKind::MySQL) {
if sf.data_type.to_ascii_uppercase().contains("TIMESTAMP WITH TIME ZONE")
|| sf.data_type.to_ascii_uppercase().contains("TIMESTAMPTZ")
{
warnings.push("PG TIMESTAMPTZ → MySQL TIMESTAMP(时区丢失)".to_string());
}
}
out.push(FieldMigration {
source_field: sf.name.clone(),
target_field: target_field_names[i].clone(),
source_type: sf.data_type.clone(),
target_type: final_type,
transform: "type_cast".to_string(),
lossy,
warnings,
});
}
Ok(out)
}
fn adjust_type_by_version(
base_type: &str,
db: TargetDbKind,
tgt_v: &DatabaseVersion,
) -> String {
let upper = base_type.to_ascii_uppercase();
match db {
TargetDbKind::MySQL | TargetDbKind::MariaDB | TargetDbKind::TiDB => {
if tgt_v.major >= 8 && upper == "DATETIME" {
return "DATETIME(6)".to_string();
}
}
TargetDbKind::PostgreSQL => {
if tgt_v.major >= 10 && upper.contains("SERIAL") {
let base = upper.split_whitespace().next().unwrap_or("INT");
return format!("{} GENERATED BY DEFAULT AS IDENTITY", base);
}
}
TargetDbKind::SQLite => {
if tgt_v.major >= 3 && tgt_v.minor >= 38 && upper == "JSON" {
return "JSONB".to_string();
}
}
_ => {}
}
base_type.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::models::{Field, Index};
fn mk(name: &str, ty: &str) -> Field {
Field {
name: name.to_string(),
data_type: ty.to_string(),
length: None,
nullable: true,
default_value: None,
primary_key: false,
auto_increment: false,
}
}
#[test]
fn test_default_versions() {
let mysql_v = default_version_for(TargetDbKind::MySQL);
assert_eq!(mysql_v.major, 8);
let pg_v = default_version_for(TargetDbKind::PostgreSQL);
assert_eq!(pg_v.major, 16);
}
#[test]
fn test_migration_direction_same_db() {
let c = MigrationConfig {
source_db: TargetDbKind::MySQL,
target_db: TargetDbKind::MySQL,
source_version: Some(DatabaseVersion::new(5, 7, 40)),
target_version: Some(DatabaseVersion::new(8, 0, 32)),
..Default::default()
};
assert_eq!(c.direction(), MigrationDirection::SameDbCrossVersion);
}
#[test]
fn test_migration_direction_cross_db_same_version() {
let c = MigrationConfig {
source_db: TargetDbKind::MySQL,
target_db: TargetDbKind::PostgreSQL,
source_version: Some(DatabaseVersion::new(8, 0, 0)),
target_version: Some(DatabaseVersion::new(8, 0, 0)),
..Default::default()
};
assert_eq!(c.direction(), MigrationDirection::CrossDbSameVersion);
}
#[test]
fn test_migration_direction_cross_db_cross_version() {
let c = MigrationConfig {
source_db: TargetDbKind::MySQL,
target_db: TargetDbKind::PostgreSQL,
source_version: Some(DatabaseVersion::new(5, 7, 40)),
target_version: Some(DatabaseVersion::new(16, 2, 0)),
..Default::default()
};
assert_eq!(c.direction(), MigrationDirection::CrossDbCrossVersion);
}
#[test]
fn test_migration_direction_same_db_same_version() {
let c = MigrationConfig {
source_db: TargetDbKind::MySQL,
target_db: TargetDbKind::MySQL,
source_version: Some(DatabaseVersion::new(8, 0, 32)),
target_version: Some(DatabaseVersion::new(8, 0, 32)),
..Default::default()
};
assert_eq!(c.direction(), MigrationDirection::SameDbSameVersion);
}
#[test]
fn test_mysql5_to_8_datetime_upgrade() {
let mut f = mk("created_at", "DATETIME");
let src_v = DatabaseVersion::new(5, 7, 40);
let tgt_v = DatabaseVersion::new(8, 0, 32);
upgrade_field_type(&mut f, TargetDbKind::MySQL, &src_v, &tgt_v);
assert_eq!(f.data_type, "DATETIME(6)");
}
#[test]
fn test_pg9_to_16_serial_to_identity() {
let mut f = mk("id", "SERIAL");
let src_v = DatabaseVersion::new(9, 6, 0);
let tgt_v = DatabaseVersion::new(16, 0, 0);
upgrade_field_type(&mut f, TargetDbKind::PostgreSQL, &src_v, &tgt_v);
assert!(f.data_type.contains("IDENTITY"));
}
#[test]
fn test_sqlite_3_20_to_3_45_json_to_jsonb() {
let mut f = mk("data", "JSON");
let src_v = DatabaseVersion::new(3, 20, 0);
let tgt_v = DatabaseVersion::new(3, 45, 0);
upgrade_field_type(&mut f, TargetDbKind::SQLite, &src_v, &tgt_v);
assert_eq!(f.data_type, "JSONB");
}
#[test]
fn test_no_upgrade_when_target_not_higher() {
let mut f = mk("created_at", "DATETIME");
let src_v = DatabaseVersion::new(8, 0, 32);
let tgt_v = DatabaseVersion::new(5, 7, 0);
upgrade_field_type(&mut f, TargetDbKind::MySQL, &src_v, &tgt_v);
assert_eq!(f.data_type, "DATETIME");
}
#[test]
fn test_migrate_table_mysql5_to_pg16() {
let mig = DataMigrator::new();
let table = TableSchema {
name: "orders".to_string(),
fields: vec![
mk("id", "INT"),
mk("amount", "DECIMAL(10,2)"),
mk("created_at", "DATETIME"),
mk("data", "JSON"),
],
indexes: vec![],
foreign_keys: vec![],
};
let config = MigrationConfig {
source_db: TargetDbKind::MySQL,
target_db: TargetDbKind::PostgreSQL,
source_version: Some(DatabaseVersion::new(5, 7, 40)),
target_version: Some(DatabaseVersion::new(16, 2, 0)),
..Default::default()
};
let result = mig.migrate_table(&table, &config).unwrap();
assert_eq!(result.direction, MigrationDirection::CrossDbCrossVersion);
assert_eq!(result.fields_total, 4);
assert_eq!(result.fields_mapped, 4);
assert!(result.warnings.iter().any(|w| w.contains("TIMESTAMP") || w.contains("JSONB")));
}
#[test]
fn test_migrate_with_unmapped_field() {
let mig = DataMigrator::new();
let table = TableSchema {
name: "users".to_string(),
fields: vec![
mk("id", "INT"),
mk("foo_unknown", "VARCHAR(64)"),
],
indexes: vec![],
foreign_keys: vec![],
};
let linker = FieldLinker::new();
let links = linker.link(&table.fields, &[]);
assert_eq!(links[1].match_kind, LinkKind::Unmatched);
}
#[test]
fn test_auto_field_link_semantic_match() {
let mig = DataMigrator::new();
let src = vec![mk("id", "INT"), mk("created_at", "DATETIME")];
let tgt = vec![mk("id", "INT"), mk("create_time", "TIMESTAMP")];
let linker = FieldLinker::new();
let links = linker.link(&src, &tgt);
assert!(links.iter().any(|l| l.match_kind == LinkKind::Semantic));
}
#[test]
fn test_manual_field_map_overrides() {
let mig = DataMigrator::new();
let mut manual = HashMap::new();
manual.insert("u_email".to_string(), "user_email".to_string());
let config = MigrationConfig {
source_db: TargetDbKind::MySQL,
target_db: TargetDbKind::PostgreSQL,
source_version: Some(DatabaseVersion::new(8, 0, 0)),
target_version: Some(DatabaseVersion::new(16, 0, 0)),
manual_field_map: manual,
..Default::default()
};
let table = TableSchema {
name: "users".to_string(),
fields: vec![mk("u_email", "VARCHAR(128)")],
indexes: vec![],
foreign_keys: vec![],
};
let result = mig.migrate_table(&table, &config).unwrap();
assert_eq!(result.field_migrations[0].target_field, "u_email"); }
#[test]
fn test_ddl_generation_basic() {
let mig = DataMigrator::new();
let table = TableSchema {
name: "orders".to_string(),
fields: vec![mk("id", "INT"), mk("name", "VARCHAR(64)")],
indexes: vec![Index {
name: "idx_name".to_string(),
fields: vec!["name".to_string()],
unique: false,
}],
foreign_keys: vec![],
};
let config = MigrationConfig {
source_db: TargetDbKind::MySQL,
target_db: TargetDbKind::PostgreSQL,
source_version: Some(DatabaseVersion::new(8, 0, 0)),
target_version: Some(DatabaseVersion::new(16, 0, 0)),
..Default::default()
};
let result = mig.migrate_table(&table, &config).unwrap();
assert!(result.ddl.contains("CREATE TABLE"));
assert!(result.ddl.contains("id"));
}
#[test]
fn test_lossy_count_mysql_to_sqlite() {
let mig = DataMigrator::new();
let table = TableSchema {
name: "events".to_string(),
fields: vec![
mk("id", "INT"),
mk("created_at", "DATETIME"), mk("payload", "JSON"), ],
indexes: vec![],
foreign_keys: vec![],
};
let config = MigrationConfig {
source_db: TargetDbKind::MySQL,
target_db: TargetDbKind::SQLite,
source_version: Some(DatabaseVersion::new(8, 0, 0)),
target_version: Some(DatabaseVersion::new(3, 45, 0)),
..Default::default()
};
let result = mig.migrate_table(&table, &config).unwrap();
assert!(result.lossy_conversions >= 1);
}
#[test]
fn test_success_rate() {
let r = TableMigrationResult {
table_name: "x".to_string(),
direction: MigrationDirection::CrossDbSameVersion,
source_db: TargetDbKind::MySQL,
target_db: TargetDbKind::PostgreSQL,
source_version: "8.0.0".to_string(),
target_version: "16.0.0".to_string(),
fields_total: 10,
fields_mapped: 9,
fields_unmapped: 1,
lossy_conversions: 0,
warnings: vec![],
field_migrations: vec![],
ddl: "".to_string(),
started_at: "".to_string(),
finished_at: "".to_string(),
elapsed_ms: 0,
};
assert!((r.success_rate() - 0.9).abs() < 0.001);
}
#[test]
fn test_effective_versions() {
let c = MigrationConfig::default();
assert_eq!(c.effective_source_version().major, 8);
assert_eq!(c.effective_target_version().major, 16);
}
}