use postg::config::{Config, Engine};
use postg::engine::Postg;
use std::time::Duration;
use tokio::time::sleep;
#[tokio::test]
#[ignore]
async fn test_spock_multi_master_replication() {
let tmp_path = std::path::PathBuf::from("/tmp/spock_test_debug");
let _ = std::fs::remove_dir_all(&tmp_path);
std::fs::create_dir_all(&tmp_path).unwrap();
let data_dir_a = tmp_path.join("node_a");
let data_dir_b = tmp_path.join("node_b");
let config_a = Config {
engine: Engine::PostgresqlSpock,
data_dir: data_dir_a,
temporary: false,
..Config::default()
};
let mut node_a = Postg::start(config_a)
.await
.expect("Failed to start Node A");
let config_b = Config {
engine: Engine::PostgresqlSpock,
data_dir: data_dir_b,
temporary: false,
..Config::default()
};
let mut node_b = Postg::start(config_b.clone())
.await
.expect("Failed to start Node B");
let pool_a = sqlx::PgPool::connect(&node_a.connection_string())
.await
.unwrap();
let pool_b = sqlx::PgPool::connect(&node_b.connection_string())
.await
.unwrap();
let schema_sql = "CREATE TABLE messages (id serial PRIMARY KEY, content text NOT NULL);";
sqlx::query(schema_sql).execute(&pool_a).await.unwrap();
sqlx::query(schema_sql).execute(&pool_b).await.unwrap();
sqlx::query("CREATE EXTENSION IF NOT EXISTS spock;")
.execute(&pool_a)
.await
.unwrap();
sqlx::query("CREATE EXTENSION IF NOT EXISTS spock;")
.execute(&pool_b)
.await
.unwrap();
let dsn_a = node_a.connection_string();
let dsn_b = node_b.connection_string();
sqlx::query("SELECT spock.node_create(node_name := 'node_a'::name, dsn := $1)")
.bind(&dsn_a)
.execute(&pool_a)
.await
.unwrap();
sqlx::query("SELECT spock.node_create(node_name := 'node_b'::name, dsn := $1)")
.bind(&dsn_b)
.execute(&pool_b)
.await
.unwrap();
sqlx::query("SELECT spock.repset_add_table('default', 'messages', true);")
.execute(&pool_a)
.await
.unwrap();
sqlx::query("SELECT spock.repset_add_table('default', 'messages', true);")
.execute(&pool_b)
.await
.unwrap();
sqlx::query(
"SELECT spock.sub_create(subscription_name := 'sub_a_to_b'::name, provider_dsn := $1)",
)
.bind(&dsn_b)
.execute(&pool_a)
.await
.unwrap();
sqlx::query(
"SELECT spock.sub_create(subscription_name := 'sub_b_to_a'::name, provider_dsn := $1)",
)
.bind(&dsn_a)
.execute(&pool_b)
.await
.unwrap();
sleep(Duration::from_millis(2000)).await;
sqlx::query("INSERT INTO messages (id, content) VALUES (1, 'hello from A')")
.execute(&pool_a)
.await
.unwrap();
let mut replicated_to_b = false;
for _ in 0..20 {
let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages WHERE id = 1")
.fetch_one(&pool_b)
.await
.unwrap();
if count.0 == 1 {
replicated_to_b = true;
break;
}
sleep(Duration::from_millis(200)).await;
}
assert!(
replicated_to_b,
"Data from Node A did not replicate to Node B"
);
sqlx::query("INSERT INTO messages (id, content) VALUES (2, 'hello from B')")
.execute(&pool_b)
.await
.unwrap();
let mut replicated_to_a = false;
for _ in 0..20 {
let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages WHERE id = 2")
.fetch_one(&pool_a)
.await
.unwrap();
if count.0 == 1 {
replicated_to_a = true;
break;
}
sleep(Duration::from_millis(200)).await;
}
assert!(
replicated_to_a,
"Data from Node B did not replicate to Node A"
);
sqlx::query("UPDATE messages SET content = 'updated from A' WHERE id = 1")
.execute(&pool_a)
.await
.unwrap();
let mut update_replicated = false;
for _ in 0..20 {
let content: (String,) = sqlx::query_as("SELECT content FROM messages WHERE id = 1")
.fetch_one(&pool_b)
.await
.unwrap();
if content.0 == "updated from A" {
update_replicated = true;
break;
}
sleep(Duration::from_millis(200)).await;
}
assert!(
update_replicated,
"UPDATE from Node A did not replicate to Node B"
);
sqlx::query("DELETE FROM messages WHERE id = 2")
.execute(&pool_b)
.await
.unwrap();
let mut delete_replicated = false;
for _ in 0..20 {
let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages WHERE id = 2")
.fetch_one(&pool_a)
.await
.unwrap();
if count.0 == 0 {
delete_replicated = true;
break;
}
sleep(Duration::from_millis(200)).await;
}
assert!(
delete_replicated,
"DELETE from Node B did not replicate to Node A"
);
pool_b.close().await;
node_b.stop().await.unwrap();
sqlx::query("INSERT INTO messages (id, content) VALUES (3, 'inserted while B was offline')")
.execute(&pool_a)
.await
.unwrap();
let config_b_restarted = node_b.config().clone();
node_b = Postg::start(config_b_restarted)
.await
.expect("Failed to restart Node B");
let pool_b = sqlx::PgPool::connect(&node_b.connection_string())
.await
.unwrap();
let mut offline_sync = false;
for _ in 0..40 {
let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages WHERE id = 3")
.fetch_one(&pool_b)
.await
.unwrap_or((0,));
if count.0 == 1 {
offline_sync = true;
break;
}
sleep(Duration::from_millis(200)).await;
}
assert!(
offline_sync,
"Node B did not sync missed data after coming back online"
);
sqlx::query("UPDATE messages SET content = 'conflict A' WHERE id = 3")
.execute(&pool_a)
.await
.unwrap();
sqlx::query("UPDATE messages SET content = 'conflict B' WHERE id = 3")
.execute(&pool_b)
.await
.unwrap();
sleep(Duration::from_millis(1000)).await;
sqlx::query("ALTER TABLE messages ADD COLUMN created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP")
.execute(&pool_a)
.await
.unwrap();
let mut ddl_replicated = false;
for _ in 0..20 {
let result = sqlx::query("SELECT created_at FROM messages LIMIT 1")
.execute(&pool_b)
.await;
if result.is_ok() {
ddl_replicated = true;
break;
}
sleep(Duration::from_millis(500)).await;
}
assert!(
ddl_replicated,
"DDL (ALTER TABLE) did not replicate from Node A to Node B"
);
pool_a.close().await;
pool_b.close().await;
node_a.stop().await.unwrap();
node_b.stop().await.unwrap();
}