arcula 2.0.0

Arcula - MongoDB database synchronization tool
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
use std::env;
use std::process::{Command, Stdio};
use std::thread;
use std::time::Duration;

use ::mongodb::bson::{doc, Document};
use ::mongodb::Client;
use anyhow::Result;
use arcula::config::{Environment, MongoConfig};
use arcula::core::sync::{SyncConfig, SyncOptions};
use arcula::utils::mongodb;

// This file contains integration tests that use real MongoDB instances
// It uses Docker to spin up temporary MongoDB containers for testing

// Get the IP address of a Docker container by name
fn get_container_ip(container_name: &str) -> Result<String> {
    let output = Command::new("docker")
        .args([
            "inspect",
            "-f",
            "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
            container_name,
        ])
        .output()?;

    if !output.status.success() {
        return Err(anyhow::anyhow!("Failed to get container IP address"));
    }

    let ip = String::from_utf8(output.stdout)?.trim().to_string();

    if ip.is_empty() {
        return Err(anyhow::anyhow!("Container IP address not found"));
    }

    Ok(ip)
}

// Use a function to generate unique container names for each test
fn generate_container_names() -> (String, String) {
    // Generate a unique suffix based on timestamp and random number
    let suffix = format!(
        "{}_{}",
        std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .unwrap_or_default()
            .as_secs(),
        rand::random::<u16>()
    );

    (
        format!("mongo_importer_test_source_{}", suffix),
        format!("mongo_importer_test_target_{}", suffix),
    )
}

// Environment variables for CI environment
const ENV_MONGO_SOURCE_URI: &str = "TEST_MONGO_SOURCE_URI";
const ENV_MONGO_TARGET_URI: &str = "TEST_MONGO_TARGET_URI";

// Setup function to start MongoDB containers with unique names and get their IPs
fn setup_mongodb_containers() -> Result<((String, String), (String, String))> {
    // Check if Docker is available
    let docker_check = Command::new("docker").arg("--version").output()?;

    if !docker_check.status.success() {
        eprintln!("Docker is not available.");
        return Err(anyhow::anyhow!("Docker is not available"));
    }

    // Generate unique container names for this test run
    let (container_name1, container_name2) = generate_container_names();

    // Start the source MongoDB container
    let start_source = Command::new("docker")
        .args([
            "run",
            "--rm",
            "-d",
            "--name",
            &container_name1,
            "mongo:latest",
        ])
        .stdout(Stdio::null())
        .status()?;

    if !start_source.success() {
        return Err(anyhow::anyhow!("Failed to start source MongoDB container"));
    }

    // Start the target MongoDB container
    let start_target = Command::new("docker")
        .args([
            "run",
            "--rm",
            "-d",
            "--name",
            &container_name2,
            "mongo:latest",
        ])
        .stdout(Stdio::null())
        .status()?;

    if !start_target.success() {
        // Clean up the first container if the second fails
        let _ = Command::new("docker")
            .args(["rm", "-f", &container_name1])
            .stdout(Stdio::null())
            .status();
        return Err(anyhow::anyhow!("Failed to start target MongoDB container"));
    }

    // Wait for MongoDB to be ready
    println!("Waiting for MongoDB containers to be ready...");
    thread::sleep(Duration::from_secs(5));

    // Get container IP addresses
    let ip1 = get_container_ip(&container_name1)?;
    let ip2 = get_container_ip(&container_name2)?;

    println!("MongoDB containers running at IPs {} and {}", ip1, ip2);

    Ok(((container_name1, container_name2), (ip1, ip2)))
}

// Teardown function to stop and remove MongoDB containers
fn teardown_mongodb_containers(container_names: &(String, String)) -> Result<()> {
    // Stop and remove the containers
    let _ = Command::new("docker")
        .args(["rm", "-f", &container_names.0])
        .stdout(Stdio::null())
        .status();

    let _ = Command::new("docker")
        .args(["rm", "-f", &container_names.1])
        .stdout(Stdio::null())
        .status();

    Ok(())
}

/// Create MongoDB configurations for testing
///
/// This function will use connection strings from environment variables if they exist,
/// otherwise it will use the provided container IPs.
fn get_test_configs(ips: Option<(String, String)>) -> (MongoConfig, MongoConfig) {
    // Check if connection strings are provided via environment variables
    let source_uri = env::var(ENV_MONGO_SOURCE_URI).unwrap_or_else(|_| {
        let ip = ips
            .as_ref()
            .map(|(ip, _)| ip.clone())
            .unwrap_or_else(|| "localhost".to_string());
        format!("mongodb://{}:27017", ip)
    });

    let target_uri = env::var(ENV_MONGO_TARGET_URI).unwrap_or_else(|_| {
        let ip = ips
            .as_ref()
            .map(|(_, ip)| ip.clone())
            .unwrap_or_else(|| "localhost".to_string());
        format!("mongodb://{}:27017", ip)
    });

    let source_config = MongoConfig {
        connection_string: source_uri,
        environment: Environment::new("TEST_SOURCE"),
    };

    let target_config = MongoConfig {
        connection_string: target_uri,
        environment: Environment::new("TEST_TARGET"),
    };

    (source_config, target_config)
}

// Helper function to create test data in source MongoDB
async fn create_test_data(config: &MongoConfig, db_name: &str) -> Result<()> {
    // Use the MongoDB client to create test data
    let client_options = config.get_client_options().await?;
    let client = Client::with_options(client_options)?;
    let db = client.database(db_name);
    let collection = db.collection::<Document>("test_collection");

    // Create test documents
    for i in 0..10 {
        let doc = doc! {
            "test_field": format!("test_value_{}", i),
            "test_number": i
        };
        collection.insert_one(doc).await?;
    }

    Ok(())
}

// Helper function to verify data was synced correctly
async fn verify_synced_data(config: &MongoConfig, db_name: &str) -> Result<bool> {
    // Use the MongoDB client to verify the data
    let client_options = config.get_client_options().await?;
    let client = Client::with_options(client_options)?;
    let db = client.database(db_name);
    let collection = db.collection::<Document>("test_collection");

    // Count documents
    let count = collection.count_documents(doc! {}).await?;

    Ok(count == 10)
}

// Test MongoDB connection
#[tokio::test]
async fn test_mongodb_connection() -> Result<()> {
    // Check if we have MongoDB URIs configured in environment
    let external_mongo =
        env::var(ENV_MONGO_SOURCE_URI).is_ok() && env::var(ENV_MONGO_TARGET_URI).is_ok();

    // Container names and IPs to be used for cleanup if needed
    let mut container_info = None;

    // Setup Docker containers if needed
    if !external_mongo {
        match setup_mongodb_containers() {
            Ok((container_names, ips)) => {
                container_info = Some((container_names, ips));
            }
            Err(e) => {
                eprintln!("Error setting up MongoDB containers: {}", e);
                return Err(anyhow::anyhow!(
                    "Failed to set up MongoDB containers: {}",
                    e
                ));
            }
        }
    }

    // Run the test
    let (source_config, target_config) =
        get_test_configs(container_info.as_ref().map(|(_, ips)| ips.clone()));

    // Test that we can connect to both MongoDB instances
    let source_dbs = mongodb::list_databases(&source_config).await?;
    let target_dbs = mongodb::list_databases(&target_config).await?;

    println!("Source DBs: {:?}", source_dbs);
    println!("Target DBs: {:?}", target_dbs);

    // Ensure both MongoDB instances are running
    assert!(source_dbs.contains(&"admin".to_string()));
    assert!(target_dbs.contains(&"admin".to_string()));

    // Teardown MongoDB containers if we created them
    if !external_mongo {
        if let Some((container_names, _)) = container_info {
            teardown_mongodb_containers(&container_names)?;
        }
    }

    Ok(())
}

// Test export and import functionality
#[tokio::test]
async fn test_export_import() -> Result<()> {
    // Check if we have MongoDB URIs configured in environment
    let external_mongo =
        env::var(ENV_MONGO_SOURCE_URI).is_ok() && env::var(ENV_MONGO_TARGET_URI).is_ok();

    // Container names and IPs to be used for cleanup if needed
    let mut container_info = None;

    // Setup Docker containers if needed
    if !external_mongo {
        match setup_mongodb_containers() {
            Ok((container_names, ips)) => {
                container_info = Some((container_names, ips));
            }
            Err(e) => {
                eprintln!("Error setting up MongoDB containers: {}", e);
                return Err(anyhow::anyhow!(
                    "Failed to set up MongoDB containers: {}",
                    e
                ));
            }
        }
    }

    // Get MongoDB configs
    let (source_config, target_config) =
        get_test_configs(container_info.as_ref().map(|(_, ips)| ips.clone()));

    // Create test database and collection
    let test_db = "test_db";
    create_test_data(&source_config, test_db).await?;

    // Create temporary directory for the export/import
    let temp_dir = tempfile::tempdir()?;
    let temp_path = temp_dir.path();

    // Export the database
    let export_result = mongodb::export_database(&source_config, test_db, temp_path).await;
    assert!(export_result.is_ok());

    // Import the database to the target
    let import_result =
        mongodb::import_database(&target_config, test_db, temp_path, true, false).await;
    assert!(import_result.is_ok());

    // Verify the data was imported correctly
    let verification = verify_synced_data(&target_config, test_db).await?;
    assert!(verification);

    // Teardown MongoDB containers if we created them
    if !external_mongo {
        if let Some((container_names, _)) = container_info {
            teardown_mongodb_containers(&container_names)?;
        }
    }

    Ok(())
}

// Test backup and restore functionality
#[tokio::test]
async fn test_backup_restore() -> Result<()> {
    // Check if we have MongoDB URIs configured in environment
    let external_mongo =
        env::var(ENV_MONGO_SOURCE_URI).is_ok() && env::var(ENV_MONGO_TARGET_URI).is_ok();

    // Container names and IPs to be used for cleanup if needed
    let mut container_info = None;

    // Setup Docker containers if needed
    if !external_mongo {
        match setup_mongodb_containers() {
            Ok((container_names, ips)) => {
                container_info = Some((container_names, ips));
            }
            Err(e) => {
                eprintln!("Error setting up MongoDB containers: {}", e);
                return Err(anyhow::anyhow!(
                    "Failed to set up MongoDB containers: {}",
                    e
                ));
            }
        }
    }

    // Get MongoDB configs
    let (source_config, _) = get_test_configs(container_info.as_ref().map(|(_, ips)| ips.clone()));

    // Create test database and collection
    let test_db = "backup_test_db";
    create_test_data(&source_config, test_db).await?;

    // Create a backup
    let backup_result = mongodb::create_backup(&source_config, test_db).await;
    assert!(backup_result.is_ok());
    let backup_path = backup_result.unwrap();

    // Clear the database
    let client_options = source_config.get_client_options().await?;
    let client = Client::with_options(client_options)?;
    client.database(test_db).drop().await?;

    // Restore from backup
    let restore_result = mongodb::restore_backup(&source_config, test_db, &backup_path).await;
    assert!(restore_result.is_ok());

    // Verify the data was restored correctly
    let verification = verify_synced_data(&source_config, test_db).await?;
    assert!(verification);

    // Teardown MongoDB containers if we created them
    if !external_mongo {
        if let Some((container_names, _)) = container_info {
            teardown_mongodb_containers(&container_names)?;
        }
    }

    Ok(())
}

// Test the full sync operation
#[tokio::test]
async fn test_full_sync_operation() -> Result<()> {
    // Check if we have MongoDB URIs configured in environment
    let external_mongo =
        env::var(ENV_MONGO_SOURCE_URI).is_ok() && env::var(ENV_MONGO_TARGET_URI).is_ok();

    // Container names and IPs to be used for cleanup if needed
    let mut container_info = None;

    // Setup Docker containers if needed
    if !external_mongo {
        match setup_mongodb_containers() {
            Ok((container_names, ips)) => {
                container_info = Some((container_names, ips));
            }
            Err(e) => {
                eprintln!("Error setting up MongoDB containers: {}", e);
                return Err(anyhow::anyhow!(
                    "Failed to set up MongoDB containers: {}",
                    e
                ));
            }
        }
    }

    // Get MongoDB configs
    let (source_config, target_config) =
        get_test_configs(container_info.as_ref().map(|(_, ips)| ips.clone()));

    // Create test database and collection
    let source_db = "sync_source_db";
    let target_db = "sync_target_db";
    create_test_data(&source_config, source_db).await?;

    // Create sync config
    let sync_config = SyncConfig {
        source_env: source_config.environment.clone(),
        target_env: target_config.environment.clone(),
        source_db: source_db.to_string(),
        target_db: target_db.to_string(),
        options: SyncOptions {
            create_backup: true,
            drop_collections: true,
            clear_collections: false,
        },
    };

    // Set environment variables for the config
    env::set_var("MONGO_TEST_SOURCE_URI", &source_config.connection_string);
    env::set_var("MONGO_TEST_TARGET_URI", &target_config.connection_string);

    // Perform the sync
    let sync_result = arcula::core::sync::perform_sync(sync_config).await;
    assert!(sync_result.is_ok());

    // Verify the data was synced correctly
    let verification = verify_synced_data(&target_config, target_db).await?;
    assert!(verification);

    // Clean up environment variables
    env::remove_var("MONGO_TEST_SOURCE_URI");
    env::remove_var("MONGO_TEST_TARGET_URI");

    // Teardown MongoDB containers if we created them
    if !external_mongo {
        if let Some((container_names, _)) = container_info {
            teardown_mongodb_containers(&container_names)?;
        }
    }

    Ok(())
}