mod index;
mod syntax;
use std::collections::{BTreeMap, BTreeSet};
use index::{VaultIndex, fragment_key};
use nanoid::nanoid;
use serde_json::Value as JsonValue;
use syntax::{Wikilink, markdown_label, prepare_anchors, rewrite_wikilinks};
use super::{
super::{
ImportAssetRef, ImportedAssetMetadata, IndexedArchive, add_asset_batch, doc_snapshot_with_id_hints, empty_batch,
file_name, frontmatter_meta, has_emitted_assets, hash_bytes, imported_asset_metadata, is_protected,
linked_page_reference, normalize_import_path, push_skipped_doc_warning, read_archive, register_page_path,
rewrite_link_references, rewrite_markdown_lines, rewrite_non_image_embeds_to_attachments, split_leading_emoji,
strip_extension, strip_frontmatter,
},
folder_hierarchy_deltas,
};
use crate::{
FolderHierarchyDelta, ImportBatch, ImportBatchLimits, ImportCursor, ImportError, ImportOptions, ImportProgress,
ImportProvider, ImportResult, ImportSource, ImportWarning, ImportedDocMeta, ImportedDocSnapshot, ImportedIcon,
ImportedIconData,
};
pub(super) struct ObsidianProvider;
impl ImportProvider for ObsidianProvider {
fn format(&self) -> &'static str {
"obsidian"
}
fn create_cursor(
&self,
source: ImportSource,
options: ImportOptions,
limits: ImportBatchLimits,
) -> ImportResult<Box<dyn ImportCursor>> {
let preserve_root_folders = matches!(&source, ImportSource::DirectoryPath(_));
let archive = read_archive(source, false, &options)?;
Ok(Box::new(ObsidianImportCursor::new(
archive,
options,
limits,
preserve_root_folders,
)?))
}
}
struct ObsidianDocRef {
entry_index: usize,
doc_id: String,
content: String,
block_id_hint_token: String,
title: String,
icon: Option<String>,
meta: Option<ImportedDocMeta>,
}
struct PendingReference {
page_id: String,
block_id: Option<String>,
title: String,
}
struct ObsidianImportCursor {
archive: IndexedArchive,
docs: Vec<ObsidianDocRef>,
assets: Vec<ImportAssetRef>,
asset_metadata: BTreeMap<String, ImportedAssetMetadata>,
page_ids_by_path: BTreeMap<String, String>,
blob_ids_by_path: BTreeMap<String, String>,
vault: VaultIndex,
options: ImportOptions,
limits: ImportBatchLimits,
folder_root_to_strip: Option<String>,
next_doc: usize,
emitted_assets: BTreeSet<usize>,
emitted_final: bool,
completed: usize,
total: usize,
entry_id: Option<String>,
}
impl ObsidianImportCursor {
fn new(
archive: IndexedArchive,
options: ImportOptions,
limits: ImportBatchLimits,
preserve_root_folders: bool,
) -> ImportResult<Self> {
let mut docs = Vec::new();
let mut assets = Vec::new();
let mut asset_metadata = BTreeMap::new();
let mut page_ids_by_path = BTreeMap::new();
let mut blob_ids_by_path = BTreeMap::new();
let mut vault = VaultIndex::default();
for (entry_index, entry) in archive.entries.iter().enumerate() {
let lower_path = entry.meta.path.to_lowercase();
if lower_path == ".obsidian/app.json" || lower_path.ends_with("/.obsidian/app.json") {
let bytes = archive.read_entry(entry)?;
vault.set_attachment_folder(attachment_folder(&entry.meta.path, &bytes));
continue;
}
if lower_path.starts_with(".obsidian/") || lower_path.contains("/.obsidian/") {
continue;
}
if lower_path.ends_with(".md") {
let doc_id = nanoid!();
let bytes = archive.read_entry(entry)?;
let markdown = String::from_utf8_lossy(&bytes);
let (content, frontmatter) = strip_frontmatter(&markdown);
let raw_title = frontmatter
.title
.clone()
.unwrap_or_else(|| strip_extension(file_name(&entry.meta.path)).to_string());
let (title, icon) = split_leading_emoji(&raw_title);
let block_id_hint_token = nanoid!();
let (content, anchors) = prepare_anchors(&content, &doc_id, &block_id_hint_token);
register_page_path(&mut page_ids_by_path, &entry.meta.path, &doc_id);
vault.register_page(&entry.meta.path, &doc_id);
vault.register_page_alias(&raw_title, &doc_id);
vault.register_page_alias(&title, &doc_id);
vault.register_anchors(&doc_id, anchors);
let meta = frontmatter_meta(&frontmatter, &title);
docs.push(ObsidianDocRef {
entry_index,
doc_id: doc_id.clone(),
content,
block_id_hint_token,
title,
icon,
meta,
});
} else {
let blob_id = if entry.meta.uncompressed_size <= limits.max_blob_bytes {
let blob_id = hash_bytes(&archive.read_entry(entry)?);
blob_ids_by_path.insert(entry.meta.path.clone(), blob_id.clone());
vault.register_asset(&entry.meta.path, &blob_id);
asset_metadata.insert(blob_id.clone(), imported_asset_metadata(entry));
blob_id
} else {
String::new()
};
assets.push(ImportAssetRef { entry_index, blob_id });
}
}
if docs.is_empty() {
return Err(ImportError::InvalidSource(
"No Markdown pages found in the folder".to_string(),
));
}
let total = docs.len();
let entry_id = docs.first().map(|doc| doc.doc_id.clone());
let folder_root_to_strip = if preserve_root_folders {
None
} else {
docs
.first()
.and_then(|doc| archive.entries[doc.entry_index].meta.path.split('/').next())
.filter(|root| {
!root.is_empty()
&& docs.iter().all(|doc| {
archive.entries[doc.entry_index]
.meta
.path
.starts_with(&format!("{root}/"))
})
})
.map(ToString::to_string)
};
Ok(Self {
archive,
docs,
assets,
asset_metadata,
page_ids_by_path,
blob_ids_by_path,
vault,
options,
limits,
folder_root_to_strip,
next_doc: 0,
emitted_assets: BTreeSet::new(),
emitted_final: false,
completed: 0,
total,
entry_id,
})
}
}
impl ImportCursor for ObsidianImportCursor {
fn total(&self) -> usize {
self.total
}
fn next_batch(&mut self) -> ImportResult<Option<ImportBatch>> {
if self.completed >= self.total && self.emitted_final {
return Ok(None);
}
if self.options.cancel_after_entries == Some(self.completed) {
return Err(ImportError::Cancelled);
}
let mut batch = empty_batch(self.total);
batch.entry_id = self.entry_id.clone();
if !has_emitted_assets(&self.emitted_assets, &self.assets) {
add_asset_batch(
&self.archive,
&self.limits,
&mut self.emitted_assets,
&mut batch,
&self.assets,
)?;
batch.progress = ImportProgress {
completed: self.completed,
total: self.total,
};
return Ok(Some(batch));
}
let end = (self.next_doc + self.limits.max_docs.max(1)).min(self.docs.len());
for doc in &self.docs[self.next_doc..end] {
let entry = &self.archive.entries[doc.entry_index];
let mut references = BTreeMap::new();
let mut warnings = Vec::new();
let content = rewrite_obsidian_content(
&doc.content,
&entry.meta.path,
&doc.doc_id,
&self.vault,
&mut references,
&mut warnings,
);
batch.warnings.extend(warnings);
let mut snapshot = match doc_snapshot_with_id_hints(
&doc.doc_id,
&doc.title,
&content,
&entry.meta.path,
&self.page_ids_by_path,
&self.blob_ids_by_path,
("obsidian", &doc.block_id_hint_token),
) {
Ok(snapshot) => snapshot,
Err(ImportError::Document(error)) => {
push_skipped_doc_warning(&mut batch, &entry.meta.path, error);
continue;
}
Err(error) => return Err(error),
};
rewrite_pending_references(&mut snapshot, &references);
rewrite_non_image_embeds_to_attachments(&mut snapshot, &self.asset_metadata);
batch.docs.push(ImportedDocSnapshot {
id: doc.doc_id.clone(),
snapshot,
meta: doc.meta.clone(),
});
if let Some(icon) = &doc.icon {
batch.icons.push(ImportedIcon {
doc_id: doc.doc_id.clone(),
icon: ImportedIconData {
kind: "emoji".to_string(),
unicode: icon.clone(),
},
});
}
batch.folders.extend(obsidian_folders_for_doc_path(
&entry.meta.path,
&doc.doc_id,
self.folder_root_to_strip.as_deref(),
));
}
self.next_doc = end;
self.completed = end;
self.emitted_final = self.completed >= self.total;
batch.progress = ImportProgress {
completed: self.completed,
total: self.total,
};
batch.done = self.emitted_final;
Ok(Some(batch))
}
}
fn rewrite_obsidian_content(
markdown: &str,
current_path: &str,
current_doc_id: &str,
vault: &VaultIndex,
references: &mut BTreeMap<String, PendingReference>,
warnings: &mut Vec<ImportWarning>,
) -> String {
let mut next_reference = 0usize;
rewrite_wikilinks(markdown, |link| {
let target_doc_id = if link.path.is_empty() {
Some(current_doc_id)
} else {
vault.resolve_page(link.path, current_path)
};
if let Some(page_id) = target_doc_id {
let title = link
.alias
.or_else(|| (!link.path.is_empty()).then_some(link.path))
.or(link.fragment)
.unwrap_or("")
.to_string();
let block_id = link
.fragment
.and_then(|fragment| vault.resolve_anchor(page_id, fragment));
if link.fragment.is_some() && block_id.is_none() {
warnings.push(warning(
"unresolved_obsidian_anchor",
current_path,
format!(
"Imported {} as a page link because its heading or block anchor was not found",
link.raw
),
));
}
let token = format!("affine-import-ref://{next_reference}");
next_reference += 1;
references.insert(
token.clone(),
PendingReference {
page_id: page_id.to_string(),
block_id: block_id.map(ToString::to_string),
title: title.clone(),
},
);
return format!("[{}](<{token}>)", markdown_label(&title));
}
if link.embed
&& let Some(blob_id) = vault.resolve_asset(link.path, current_path)
{
let label = asset_label(&link);
return format!("", markdown_label(label));
}
warnings.push(warning(
if link.embed {
"unresolved_obsidian_embed"
} else {
"unresolved_obsidian_link"
},
current_path,
format!("Could not resolve {}", link.raw),
));
link.raw.to_string()
})
}
fn rewrite_pending_references(value: &mut JsonValue, references: &BTreeMap<String, PendingReference>) {
rewrite_link_references(value, |target, _| {
references.get(target).map(|reference| {
linked_page_reference(
&reference.page_id,
(!reference.title.is_empty()).then_some(reference.title.as_str()),
reference.block_id.as_deref(),
)
})
});
}
fn obsidian_folders_for_doc_path(path: &str, doc_id: &str, root_to_strip: Option<&str>) -> Vec<FolderHierarchyDelta> {
let mut parts = normalize_import_path(path)
.split('/')
.map(ToString::to_string)
.collect::<Vec<_>>();
parts.pop();
if root_to_strip.is_some_and(|root| parts.first().is_some_and(|part| part == root)) {
parts.remove(0);
}
folder_hierarchy_deltas(parts, Some(doc_id), None, ToString::to_string)
}
fn attachment_folder(config_path: &str, bytes: &[u8]) -> Option<String> {
let folder = serde_json::from_slice::<JsonValue>(bytes)
.ok()?
.get("attachmentFolderPath")?
.as_str()
.map(normalize_import_path)?;
let vault_root = normalize_import_path(
config_path
.strip_suffix(".obsidian/app.json")
.unwrap_or("")
.trim_end_matches('/'),
);
Some(normalize_import_path(&format!("{vault_root}/{folder}")))
}
fn asset_label<'a>(link: &'a Wikilink<'a>) -> &'a str {
match link.alias {
Some(alias) if !is_size_alias(alias) => alias,
_ => file_name(link.path),
}
}
fn is_size_alias(alias: &str) -> bool {
let mut parts = alias.split('x');
parts
.next()
.is_some_and(|part| !part.is_empty() && part.chars().all(|character| character.is_ascii_digit()))
&& parts
.next()
.is_none_or(|part| !part.is_empty() && part.chars().all(|character| character.is_ascii_digit()))
&& parts.next().is_none()
}
fn warning(code: &str, source_path: &str, message: String) -> ImportWarning {
ImportWarning {
code: code.to_string(),
source_path: Some(source_path.to_string()),
message,
}
}
#[cfg(test)]
mod tests {
use super::{
super::tests::{import_cursor, merge_cursor, plan_zip, temp_dir, write_file, zip},
*,
};
fn plan_obsidian_zip(bytes: impl AsRef<[u8]>) -> ImportResult<ImportBatch> {
plan_zip("obsidian", bytes)
}
fn obsidian_directory_cursor(
root: std::path::PathBuf,
options: ImportOptions,
limits: ImportBatchLimits,
) -> ImportResult<Box<dyn crate::ImportCursor>> {
import_cursor("obsidian", ImportSource::DirectoryPath(root), options, limits)
}
#[test]
fn obsidian_resolves_aliases_cross_folder_assets_and_block_links() {
let bytes = zip(&[
(
"Vault/notes/Entry.md",
b"[[Target|Alias]]\n[[Target#Target|Title]]\n[[Target#^title-anchor|Title block]]\n[[Target#Section|Heading]]\n[[Target#^anchor|Block]]\n[[Target#^list-anchor|List]]\n[Mixed `code` label](../pages/Target.md?mixed=1)\n`[[MissingInline]] [Inline page](../pages/Target.md) `\n```md\n[[MissingFence]]\n[Fenced page](../pages/Target.md)\n\n```\n\n [Indented page](../pages/Target.md?indented=1)\n\n`multiline\n[Multiline page](../pages/Target.md?multiline=1)\ncode`\n\n![[logo.png]]",
),
(
"Vault/pages/Target.md",
b"# Target ^title-anchor\n\n## Section\n\nTarget paragraph ^anchor\n\n- Item ^list-anchor",
),
("Vault/assets/logo.png", &[137, 80, 78, 71]),
]);
let batch = plan_obsidian_zip(bytes).unwrap();
let snapshot = batch
.docs
.iter()
.map(|doc| serde_json::to_string(&doc.snapshot).unwrap())
.find(|snapshot| snapshot.contains("\"title\":\"Alias\""))
.unwrap();
assert!(snapshot.contains("\"title\":\"Alias\""));
assert!(snapshot.contains("\"blockIds\""));
assert!(snapshot.contains("\"sourceId\""));
assert!(batch.folders.iter().any(|folder| folder.path == "notes"));
assert!(batch.folders.iter().any(|folder| folder.path == "pages"));
assert!(!batch.folders.iter().any(|folder| folder.path == "Vault"));
assert!(snapshot.contains("MissingInline"));
assert!(snapshot.contains("MissingFence"));
assert!(snapshot.contains("[Inline page](../pages/Target.md) "));
assert!(snapshot.contains("[Fenced page](../pages/Target.md)"));
assert!(snapshot.contains(""));
assert!(snapshot.contains("../pages/Target.md?indented=1"));
assert!(snapshot.contains("../pages/Target.md?multiline=1"));
assert!(!snapshot.contains("mixed=1"));
let referenced = batch
.docs
.iter()
.flat_map(|doc| reference_block_ids(&doc.snapshot))
.collect::<Vec<_>>();
assert_eq!(referenced.len(), 5);
assert!(
referenced
.iter()
.all(|block_id| batch.docs.iter().any(|doc| contains_block_id(&doc.snapshot, block_id)))
);
assert!(batch.warnings.is_empty());
}
fn reference_block_ids(value: &JsonValue) -> Vec<String> {
let mut block_ids = Vec::new();
match value {
JsonValue::Object(map) => {
if let Some(ids) = map.get("blockIds").and_then(JsonValue::as_array) {
block_ids.extend(ids.iter().filter_map(JsonValue::as_str).map(ToString::to_string));
}
block_ids.extend(map.values().flat_map(reference_block_ids));
}
JsonValue::Array(values) => block_ids.extend(values.iter().flat_map(reference_block_ids)),
_ => {}
}
block_ids
}
fn contains_block_id(value: &JsonValue, block_id: &str) -> bool {
match value {
JsonValue::Object(map) => {
map.get("id").and_then(JsonValue::as_str) == Some(block_id)
|| map.values().any(|value| contains_block_id(value, block_id))
}
JsonValue::Array(values) => values.iter().any(|value| contains_block_id(value, block_id)),
_ => false,
}
}
#[test]
fn obsidian_emits_assets_before_docs_and_preserves_directory_root() {
let root = temp_dir();
write_file(&root, "First/Second/Entry.md", b"![[logo.png]]");
write_file(&root, "attachments/logo.png", &[137, 80, 78, 71]);
write_file(
&root,
".obsidian/app.json",
br#"{"attachmentFolderPath":"attachments"}"#,
);
let mut cursor = obsidian_directory_cursor(
root.clone(),
ImportOptions::default(),
ImportBatchLimits {
max_docs: 1,
max_blobs: 1,
max_blob_bytes: u64::MAX,
},
)
.unwrap();
let asset_batch = cursor.next_batch().unwrap().unwrap();
let doc_batch = cursor.next_batch().unwrap().unwrap();
assert_eq!(asset_batch.blobs.len(), 1);
assert!(asset_batch.docs.is_empty());
assert_eq!(doc_batch.docs.len(), 1);
assert!(doc_batch.folders.iter().any(|folder| folder.path == "First"));
assert!(doc_batch.folders.iter().any(|folder| folder.path == "First/Second"));
std::fs::remove_dir_all(root).unwrap();
let oversized_root = temp_dir();
write_file(&oversized_root, "Entry.md", b"![[large.bin]]");
write_file(&oversized_root, "large.bin", &[0, 1, 2, 3, 4]);
let mut cursor = obsidian_directory_cursor(
oversized_root.clone(),
ImportOptions::default(),
ImportBatchLimits {
max_docs: 1,
max_blobs: 1,
max_blob_bytes: 4,
},
)
.unwrap();
let skipped_asset_batch = cursor.next_batch().unwrap().unwrap();
let oversized_doc_batch = cursor.next_batch().unwrap().unwrap();
assert!(skipped_asset_batch.blobs.is_empty());
assert!(
skipped_asset_batch
.warnings
.iter()
.any(|warning| warning.code == "skipped_asset")
);
assert!(
oversized_doc_batch
.warnings
.iter()
.any(|warning| warning.code == "unresolved_obsidian_embed")
);
assert!(
!serde_json::to_string(&oversized_doc_batch.docs[0].snapshot)
.unwrap()
.contains("blob://")
);
std::fs::remove_dir_all(oversized_root).unwrap();
}
#[test]
fn obsidian_zip_plans_wikilinks_assets_and_title_emoji() {
let bytes = zip(&[
(
"Vault/😀 Entry.md",
b"[[Target]]\n![[archive.zip]]\n",
),
("Vault/Target.md", b"target"),
("Vault/assets/logo.png", &[137, 80, 78, 71]),
("Vault/archive.zip", b"zip"),
]);
let batch = plan_obsidian_zip(bytes).unwrap();
let snapshot = serde_json::to_string(&batch.docs[0].snapshot).unwrap();
assert_eq!(batch.docs.len(), 2);
assert_eq!(batch.blobs.len(), 2);
assert_eq!(batch.icons[0].icon.unicode, "😀");
assert!(snapshot.contains("\"reference\""));
assert!(snapshot.contains("\"sourceId\""));
assert!(snapshot.contains("\"flavour\":\"affine:attachment\""));
assert!(snapshot.contains("archive.zip"));
}
#[test]
fn obsidian_directory_source_plans_without_zipping_in_memory() {
let root = temp_dir();
write_file(&root, "Vault/Entry.md", b"[[Target]]\n");
write_file(&root, "Vault/Target.md", b"target");
write_file(&root, "Vault/assets/logo.png", &[137, 80, 78, 71]);
let batch = obsidian_directory_cursor(
root.clone(),
ImportOptions::default(),
ImportBatchLimits {
max_docs: usize::MAX,
max_blobs: usize::MAX,
max_blob_bytes: u64::MAX,
},
)
.and_then(merge_cursor)
.unwrap();
assert_eq!(batch.docs.len(), 2);
assert_eq!(batch.blobs.len(), 1);
let snapshot = serde_json::to_string(&batch.docs[0].snapshot).unwrap();
assert!(snapshot.contains("\"reference\""));
assert!(snapshot.contains("\"sourceId\""));
std::fs::remove_dir_all(root).unwrap();
}
}