use std::path::PathBuf;
use surreal_sync_json::from_jsonl::{sync, Config, ConversionRule};
use surreal_sync_surreal::version::testing::SurrealDbContainer;
use surreal_sync_surreal::version::SurrealMajorVersion;
mod v2_helpers {
use serde::Deserialize;
use surrealdb::sql::Thing;
#[derive(Debug, Deserialize)]
pub struct Page {
pub id: Thing,
pub title: String,
pub parent: Thing,
}
#[derive(Debug, Deserialize)]
pub struct Block {
pub id: Thing,
pub r#type: String,
pub items: Option<Vec<String>>,
pub parent: Thing,
}
#[derive(Debug, Deserialize)]
pub struct Database {
pub id: Thing,
pub name: String,
}
#[derive(Debug, Deserialize)]
pub struct Item {
pub id: Thing,
pub name: String,
pub value: i64,
}
#[derive(Debug, Deserialize)]
pub struct LogItem {
pub id: Thing,
pub level: String,
pub target: String,
}
}
mod v3_helpers {
use surrealdb3::types::{RecordId, RecordIdKey, SurrealValue};
pub fn record_id(table: &str, key: &str) -> RecordId {
RecordId::new(table, RecordIdKey::String(key.to_string()))
}
#[derive(Debug, SurrealValue)]
#[surreal(crate = "surrealdb3::types")]
pub struct Page {
pub id: RecordId,
pub title: String,
pub parent: RecordId,
}
#[derive(Debug, SurrealValue)]
#[surreal(crate = "surrealdb3::types")]
pub struct Block {
pub id: RecordId,
pub r#type: Option<String>,
pub items: Option<Vec<String>>,
pub parent: RecordId,
}
#[derive(Debug, SurrealValue)]
#[surreal(crate = "surrealdb3::types")]
pub struct Database {
pub id: RecordId,
pub name: String,
}
#[derive(Debug, SurrealValue)]
#[surreal(crate = "surrealdb3::types")]
pub struct Item {
pub id: RecordId,
pub name: String,
pub value: i64,
}
#[derive(Debug, SurrealValue)]
#[surreal(crate = "surrealdb3::types")]
pub struct LogItem {
pub id: RecordId,
pub level: String,
pub target: String,
}
}
#[tokio::test]
async fn test_jsonl_migration_e2e() -> Result<(), Box<dyn std::error::Error>> {
tracing_subscriber::fmt()
.with_env_filter("surreal_sync_json::from_jsonl=debug,test=debug")
.try_init()
.ok();
let mut db = SurrealDbContainer::new("test-jsonl-migration-e2e");
db.start()?;
db.wait_until_ready(30)?;
let surreal_endpoint = db.ws_endpoint();
let surreal_namespace = "test_jsonl_ns";
let surreal_database = "test_jsonl_db";
let test_data_dir = PathBuf::from("tests/test_data/jsonl");
let conversion_rules = vec![
r#"type="database_id",database_id databases:database_id"#.to_string(),
r#"type="page_id",page_id pages:page_id"#.to_string(),
r#"type="block_id",block_id blocks:block_id"#.to_string(),
];
let config = Config {
sources: vec![surreal_sync_json::from_jsonl::FileSource::Local(
PathBuf::from(format!("{}/", test_data_dir.display())),
)],
files: vec![],
s3_uris: vec![],
http_uris: vec![],
id_field: "id".to_string(),
id_columns: Vec::new(),
conversion_rules,
batch_size: 1000,
dry_run: false,
schema: None,
};
match db.detected_version {
Some(SurrealMajorVersion::V3) => {
let surreal = surrealdb3::engine::any::connect(&surreal_endpoint).await?;
surreal
.signin(surrealdb3::opt::auth::Root {
username: "root".to_string(),
password: "root".to_string(),
})
.await?;
surreal
.use_ns(surreal_namespace)
.use_db(surreal_database)
.await?;
for t in ["pages", "blocks", "databases"] {
surreal.query(format!("DELETE FROM {t}")).await?;
}
let sink = surreal_sync_surreal::v3::Surreal3Sink::new(surreal.clone());
sync(&sink, config).await?;
use v3_helpers::record_id;
let mut r = surreal.query("SELECT * FROM databases ORDER BY id").await?;
let databases: Vec<v3_helpers::Database> = r.take(0)?;
assert_eq!(databases.len(), 2);
assert_eq!(databases[0].id, record_id("databases", "db1"));
assert_eq!(databases[0].name, "Documentation");
assert_eq!(databases[1].id, record_id("databases", "db2"));
assert_eq!(databases[1].name, "API Docs");
let mut r = surreal.query("SELECT * FROM pages ORDER BY id").await?;
let pages: Vec<v3_helpers::Page> = r.take(0)?;
assert_eq!(pages.len(), 3);
assert_eq!(pages[0].id, record_id("pages", "page1"));
assert_eq!(pages[0].title, "Getting Started");
assert_eq!(pages[0].parent, record_id("databases", "db1"));
assert_eq!(pages[1].id, record_id("pages", "page2"));
assert_eq!(pages[1].title, "Advanced Topics");
assert_eq!(pages[1].parent, record_id("pages", "page1"));
let mut r = surreal.query("SELECT * FROM blocks ORDER BY id").await?;
let blocks: Vec<v3_helpers::Block> = r.take(0)?;
assert_eq!(blocks.len(), 4);
assert_eq!(blocks[0].id, record_id("blocks", "block1"));
if blocks[0].r#type.is_some() {
assert_eq!(blocks[0].r#type.as_deref(), Some("paragraph"));
}
assert_eq!(blocks[0].parent, record_id("pages", "page1"));
assert_eq!(blocks[3].id, record_id("blocks", "block4"));
if blocks[3].r#type.is_some() {
assert_eq!(blocks[3].r#type.as_deref(), Some("list"));
}
assert_eq!(blocks[3].parent, record_id("blocks", "block2"));
assert_eq!(
blocks[3].items,
Some(vec![
"Item 1".to_string(),
"Item 2".to_string(),
"Item 3".to_string()
])
);
for t in ["pages", "blocks", "databases"] {
surreal.query(format!("DELETE FROM {t}")).await?;
}
}
_ => {
let surreal = surrealdb::engine::any::connect(&surreal_endpoint).await?;
surreal
.signin(surrealdb::opt::auth::Root {
username: "root",
password: "root",
})
.await?;
surreal
.use_ns(surreal_namespace)
.use_db(surreal_database)
.await?;
for t in ["pages", "blocks", "databases"] {
let _: Vec<surrealdb::sql::Thing> = surreal
.query(format!("DELETE FROM {t}"))
.await?
.take("id")
.unwrap_or_default();
}
let sink = surreal_sync_surreal::v2::Surreal2Sink::new(surreal.clone());
sync(&sink, config).await?;
let mut r = surreal.query("SELECT * FROM databases ORDER BY id").await?;
let databases: Vec<v2_helpers::Database> = r.take(0)?;
assert_eq!(databases.len(), 2);
assert_eq!(databases[0].id.to_string(), "databases:db1");
assert_eq!(databases[0].name, "Documentation");
assert_eq!(databases[1].id.to_string(), "databases:db2");
assert_eq!(databases[1].name, "API Docs");
let mut r = surreal.query("SELECT * FROM pages ORDER BY id").await?;
let pages: Vec<v2_helpers::Page> = r.take(0)?;
assert_eq!(pages.len(), 3);
assert_eq!(pages[0].id.to_string(), "pages:page1");
assert_eq!(pages[0].title, "Getting Started");
assert_eq!(pages[0].parent.to_string(), "databases:db1");
assert_eq!(pages[1].id.to_string(), "pages:page2");
assert_eq!(pages[1].title, "Advanced Topics");
assert_eq!(pages[1].parent.to_string(), "pages:page1");
let mut r = surreal.query("SELECT * FROM blocks ORDER BY id").await?;
let blocks: Vec<v2_helpers::Block> = r.take(0)?;
assert_eq!(blocks.len(), 4);
assert_eq!(blocks[0].id.to_string(), "blocks:block1");
assert_eq!(blocks[0].r#type, "paragraph");
assert_eq!(blocks[0].parent.to_string(), "pages:page1");
assert_eq!(blocks[3].id.to_string(), "blocks:block4");
assert_eq!(blocks[3].r#type, "list");
assert_eq!(blocks[3].parent.to_string(), "blocks:block2");
assert_eq!(
blocks[3].items,
Some(vec![
"Item 1".to_string(),
"Item 2".to_string(),
"Item 3".to_string()
])
);
for t in ["pages", "blocks", "databases"] {
let _: Vec<surrealdb::sql::Thing> = surreal
.query(format!("DELETE FROM {t}"))
.await?
.take("id")
.unwrap_or_default();
}
}
}
Ok(())
}
#[tokio::test]
async fn test_jsonl_conversion_rules() -> Result<(), Box<dyn std::error::Error>> {
let rule_str = r#"type="page_id",page_id pages:page_id"#;
let rule = ConversionRule::parse(rule_str)?;
assert_eq!(rule.type_field, "type");
assert_eq!(rule.type_value, "page_id");
assert_eq!(rule.id_field, "page_id");
assert_eq!(rule.target_table, "pages");
assert!(ConversionRule::parse("invalid").is_err());
assert!(ConversionRule::parse(r#"type="page_id",page_id"#).is_err());
assert!(ConversionRule::parse(r#"invalid="page_id",page_id pages:page_id"#).is_err());
Ok(())
}
#[tokio::test]
async fn test_jsonl_with_custom_id_field() -> Result<(), Box<dyn std::error::Error>> {
tracing_subscriber::fmt()
.with_env_filter("surreal_sync_json::from_jsonl=debug")
.try_init()
.ok();
let mut db = SurrealDbContainer::new("test-jsonl-custom-id");
db.start()?;
db.wait_until_ready(30)?;
let test_dir = PathBuf::from("/tmp/jsonl_test_custom_id");
std::fs::create_dir_all(&test_dir)?;
let test_file = test_dir.join("items.jsonl");
std::fs::write(
&test_file,
r#"{"item_id": "item1", "name": "Test Item 1", "value": 100}
{"item_id": "item2", "name": "Test Item 2", "value": 200}"#,
)?;
let config = Config {
sources: vec![surreal_sync_json::from_jsonl::FileSource::Local(
PathBuf::from(format!("{}/", test_dir.display())),
)],
files: vec![],
s3_uris: vec![],
http_uris: vec![],
id_field: "item_id".to_string(),
id_columns: Vec::new(),
conversion_rules: vec![],
batch_size: 1000,
dry_run: false,
schema: None,
};
match db.detected_version {
Some(SurrealMajorVersion::V3) => {
let surreal = surrealdb3::engine::any::connect(db.ws_endpoint()).await?;
surreal
.signin(surrealdb3::opt::auth::Root {
username: "root".to_string(),
password: "root".to_string(),
})
.await?;
surreal.use_ns("test_custom_id").use_db("test_db").await?;
surreal.query("DELETE FROM items").await?;
let sink = surreal_sync_surreal::v3::Surreal3Sink::new(surreal.clone());
sync(&sink, config).await?;
let mut r = surreal.query("SELECT * FROM items ORDER BY id").await?;
let items: Vec<v3_helpers::Item> = r.take(0)?;
assert_eq!(items.len(), 2);
assert_eq!(items[0].id, v3_helpers::record_id("items", "item1"));
assert_eq!(items[0].name, "Test Item 1");
assert_eq!(items[0].value, 100);
assert_eq!(items[1].id, v3_helpers::record_id("items", "item2"));
assert_eq!(items[1].name, "Test Item 2");
assert_eq!(items[1].value, 200);
}
_ => {
let surreal = surrealdb::engine::any::connect(db.ws_endpoint()).await?;
surreal
.signin(surrealdb::opt::auth::Root {
username: "root",
password: "root",
})
.await?;
surreal.use_ns("test_custom_id").use_db("test_db").await?;
let _: Vec<surrealdb::sql::Thing> = surreal
.query("DELETE FROM items")
.await?
.take("id")
.unwrap_or_default();
let sink = surreal_sync_surreal::v2::Surreal2Sink::new(surreal.clone());
sync(&sink, config).await?;
let mut r = surreal.query("SELECT * FROM items ORDER BY id").await?;
let items: Vec<v2_helpers::Item> = r.take(0)?;
assert_eq!(items.len(), 2);
assert_eq!(items[0].id.to_string(), "items:item1");
assert_eq!(items[0].name, "Test Item 1");
assert_eq!(items[0].value, 100);
assert_eq!(items[1].id.to_string(), "items:item2");
assert_eq!(items[1].name, "Test Item 2");
assert_eq!(items[1].value, 200);
}
}
std::fs::remove_dir_all(&test_dir)?;
Ok(())
}
#[tokio::test]
async fn test_jsonl_with_complex_id_field() {
tracing_subscriber::fmt()
.with_env_filter("surreal_sync_json::from_jsonl=debug")
.try_init()
.ok();
let mut db = SurrealDbContainer::new("test-jsonl-complex-id");
db.start().unwrap();
db.wait_until_ready(30).unwrap();
let test_dir = PathBuf::from("/tmp/jsonl_test_complex_id");
std::fs::create_dir_all(&test_dir).unwrap();
let test_file = test_dir.join("items.jsonl");
std::fs::write(
&test_file,
r#"{"timestamp":"2025-07-22T03:18:59.349350Z","level":"INFO","target":"surreal::env"}
{"timestamp":"2025-07-22T03:19:59.349350Z","level":"INFO","target":"surreal::env"}"#,
)
.unwrap();
let config = Config {
sources: vec![surreal_sync_json::from_jsonl::FileSource::Local(
PathBuf::from(format!("{}/", test_dir.display())),
)],
files: vec![],
s3_uris: vec![],
http_uris: vec![],
id_field: "timestamp".to_string(),
id_columns: Vec::new(),
conversion_rules: vec![],
batch_size: 1000,
dry_run: false,
schema: None,
};
match db.detected_version {
Some(SurrealMajorVersion::V3) => {
let surreal = surrealdb3::engine::any::connect(db.ws_endpoint())
.await
.unwrap();
surreal
.signin(surrealdb3::opt::auth::Root {
username: "root".to_string(),
password: "root".to_string(),
})
.await
.unwrap();
surreal
.use_ns("test_complex_id")
.use_db("test_db")
.await
.unwrap();
surreal.query("DELETE FROM items").await.unwrap();
let sink = surreal_sync_surreal::v3::Surreal3Sink::new(surreal.clone());
sync(&sink, config).await.unwrap();
let mut r = surreal.query("SELECT * FROM items").await.unwrap();
let items: Vec<v3_helpers::LogItem> = r.take(0).unwrap();
assert_eq!(items.len(), 2);
assert_eq!(items[0].level, "INFO");
assert_eq!(items[0].target, "surreal::env");
assert_eq!(items[1].level, "INFO");
assert_eq!(items[1].target, "surreal::env");
}
_ => {
let surreal = surrealdb::engine::any::connect(db.ws_endpoint())
.await
.unwrap();
surreal
.signin(surrealdb::opt::auth::Root {
username: "root",
password: "root",
})
.await
.unwrap();
surreal
.use_ns("test_complex_id")
.use_db("test_db")
.await
.unwrap();
let _: Vec<surrealdb::sql::Thing> = surreal
.query("DELETE FROM items")
.await
.unwrap()
.take("id")
.unwrap_or_default();
let sink = surreal_sync_surreal::v2::Surreal2Sink::new(surreal.clone());
sync(&sink, config).await.unwrap();
let mut r = surreal.query("SELECT * FROM items").await.unwrap();
let items: Vec<v2_helpers::LogItem> = r.take(0).unwrap();
assert_eq!(items.len(), 2);
let expected_id_0 = surrealdb::sql::Thing::from((
"items".to_string(),
"2025-07-22T03:18:59.349350Z".to_string(),
));
let expected_id_1 = surrealdb::sql::Thing::from((
"items".to_string(),
"2025-07-22T03:19:59.349350Z".to_string(),
));
assert_eq!(items[0].id, expected_id_0);
assert_eq!(items[0].level, "INFO");
assert_eq!(items[0].target, "surreal::env");
assert_eq!(items[1].id, expected_id_1);
assert_eq!(items[1].level, "INFO");
assert_eq!(items[1].target, "surreal::env");
}
}
std::fs::remove_dir_all(&test_dir).unwrap();
}
#[tokio::test]
async fn test_jsonl_composite_pk_array_record_ids() {
tracing_subscriber::fmt()
.with_env_filter("surreal_sync_json::from_jsonl=debug")
.try_init()
.ok();
let mut db = SurrealDbContainer::new("test-jsonl-composite-pk-array");
db.start().unwrap();
db.wait_until_ready(30).unwrap();
let test_dir = PathBuf::from("/tmp/jsonl_test_composite_pk_array");
std::fs::create_dir_all(&test_dir).unwrap();
let test_file = test_dir.join("ledger.jsonl");
std::fs::write(
&test_file,
r#"{"a":"x:y","b":"z","amount":1}
{"a":"x","b":"y:z","amount":2}
{"a":"a_b","b":"c","amount":3}
{"a":"a","b":"b_c","amount":4}
{"a":1,"b":2,"amount":10}
"#,
)
.unwrap();
let config = Config {
sources: vec![surreal_sync_json::from_jsonl::FileSource::Local(
PathBuf::from(format!("{}/", test_dir.display())),
)],
files: vec![],
s3_uris: vec![],
http_uris: vec![],
id_field: "id".to_string(),
id_columns: vec!["a".to_string(), "b".to_string()],
conversion_rules: vec![],
batch_size: 1000,
dry_run: false,
schema: None,
};
match db.detected_version {
Some(SurrealMajorVersion::V3) => {
use surrealdb3::types::{RecordId, RecordIdKey, SurrealValue, Value};
let surreal = surrealdb3::engine::any::connect(db.ws_endpoint())
.await
.unwrap();
surreal
.signin(surrealdb3::opt::auth::Root {
username: "root".to_string(),
password: "root".to_string(),
})
.await
.unwrap();
surreal.use_ns("test_ns").use_db("test_db").await.unwrap();
let _ = surreal.query("DELETE FROM ledger").await;
let sink = surreal_sync_surreal::v3::Surreal3Sink::new(surreal.clone());
sync(&sink, config).await.unwrap();
#[derive(Debug, SurrealValue)]
#[surreal(crate = "surrealdb3::types")]
struct LedgerRow {
id: RecordId,
amount: i64,
}
let mut r = surreal
.query("SELECT id, amount FROM ledger")
.await
.unwrap();
let rows: Vec<LedgerRow> = r.take(0).unwrap();
assert_eq!(rows.len(), 5, "array IDs must not collide: {rows:?}");
let mut amounts_by_key: Vec<(Vec<String>, i64)> = rows
.into_iter()
.map(|row| {
let key = match row.id.key {
RecordIdKey::Array(arr) => arr
.into_iter()
.map(|v| match v {
Value::String(s) => s,
Value::Number(n) => n.to_string(),
other => panic!("unexpected array id element: {other:?}"),
})
.collect(),
other => panic!("expected RecordIdKey::Array, got {other:?}"),
};
(key, row.amount)
})
.collect();
amounts_by_key.sort_by(|a, b| a.0.cmp(&b.0));
assert!(amounts_by_key
.iter()
.any(|(k, a)| { k == &vec!["x:y".to_string(), "z".to_string()] && *a == 1 }));
assert!(amounts_by_key
.iter()
.any(|(k, a)| { k == &vec!["x".to_string(), "y:z".to_string()] && *a == 2 }));
assert!(amounts_by_key
.iter()
.any(|(k, a)| { k == &vec!["a_b".to_string(), "c".to_string()] && *a == 3 }));
assert!(amounts_by_key
.iter()
.any(|(k, a)| { k == &vec!["a".to_string(), "b_c".to_string()] && *a == 4 }));
assert!(amounts_by_key
.iter()
.any(|(k, a)| { k == &vec!["1".to_string(), "2".to_string()] && *a == 10 }));
let _ = surreal.query("DELETE FROM ledger").await;
}
_ => {
use surrealdb::sql::{Id, Thing, Value};
let surreal = surrealdb::engine::any::connect(db.ws_endpoint())
.await
.unwrap();
surreal
.signin(surrealdb::opt::auth::Root {
username: "root",
password: "root",
})
.await
.unwrap();
surreal.use_ns("test_ns").use_db("test_db").await.unwrap();
let _: Vec<Thing> = surreal
.query("DELETE FROM ledger")
.await
.unwrap()
.take("id")
.unwrap_or_default();
let sink = surreal_sync_surreal::v2::Surreal2Sink::new(surreal.clone());
sync(&sink, config).await.unwrap();
#[derive(Debug, serde::Deserialize)]
struct LedgerRow {
id: Thing,
amount: i64,
}
let mut r = surreal
.query("SELECT id, amount FROM ledger")
.await
.unwrap();
let rows: Vec<LedgerRow> = r.take(0).unwrap();
assert_eq!(rows.len(), 5, "array IDs must not collide: {rows:?}");
let mut amounts_by_key: Vec<(Vec<String>, i64)> = rows
.into_iter()
.map(|row| {
let key = match row.id.id {
Id::Array(arr) => arr
.0
.into_iter()
.map(|v| match v {
Value::Strand(s) => s.as_string(),
Value::Number(n) => n.to_string(),
other => panic!("unexpected array id element: {other:?}"),
})
.collect(),
other => panic!("expected Id::Array, got {other:?}"),
};
(key, row.amount)
})
.collect();
amounts_by_key.sort_by(|a, b| a.0.cmp(&b.0));
assert!(amounts_by_key
.iter()
.any(|(k, a)| { k == &vec!["x:y".to_string(), "z".to_string()] && *a == 1 }));
assert!(amounts_by_key
.iter()
.any(|(k, a)| { k == &vec!["x".to_string(), "y:z".to_string()] && *a == 2 }));
assert!(amounts_by_key
.iter()
.any(|(k, a)| { k == &vec!["a_b".to_string(), "c".to_string()] && *a == 3 }));
assert!(amounts_by_key
.iter()
.any(|(k, a)| { k == &vec!["a".to_string(), "b_c".to_string()] && *a == 4 }));
assert!(amounts_by_key
.iter()
.any(|(k, a)| { k == &vec!["1".to_string(), "2".to_string()] && *a == 10 }));
}
}
std::fs::remove_dir_all(&test_dir).unwrap();
}