use std::collections::{BTreeMap, BTreeSet};
use nanoid::nanoid;
use super::{
ImportAssetRef, ImportBatch, ImportBatchLimits, ImportCursor, ImportDocRef, ImportError, ImportOptions,
ImportProgress, ImportResult, ImportWarning, ImportedDocMeta, ImportedDocSnapshot, IndexedArchive, add_asset_batch,
doc_snapshot, empty_batch, entry_from_indexed, file_name, folder_hierarchy_deltas, frontmatter_meta,
has_emitted_assets, hash_bytes, normalize_import_path, parse_link_span, push_skipped_doc_warning, register_csv_path,
register_page_path, resolve_path, strip_extension, strip_frontmatter, strip_notion_hash,
};
pub(super) struct NotionMarkdownImportCursor {
archive: IndexedArchive,
docs: Vec<ImportDocRef>,
assets: Vec<ImportAssetRef>,
csv_entries_by_path: BTreeMap<String, usize>,
page_ids_by_path: BTreeMap<String, String>,
blob_ids_by_path: BTreeMap<String, String>,
folders: Vec<crate::FolderHierarchyDelta>,
warnings: Vec<ImportWarning>,
options: ImportOptions,
limits: ImportBatchLimits,
next_doc: usize,
emitted_assets: BTreeSet<usize>,
emitted_final: bool,
completed: usize,
total: usize,
entry_id: Option<String>,
}
impl NotionMarkdownImportCursor {
pub(super) fn new(archive: IndexedArchive, options: ImportOptions, limits: ImportBatchLimits) -> ImportResult<Self> {
let mut docs = Vec::new();
let mut assets = Vec::new();
let mut csv_entries_by_path = BTreeMap::new();
let mut page_ids_by_path = BTreeMap::new();
let mut blob_ids_by_path = BTreeMap::new();
for (entry_index, entry) in archive.entries.iter().enumerate() {
if entry.meta.path.to_lowercase().ends_with(".md") {
let doc_id = nanoid!();
register_page_path(&mut page_ids_by_path, &entry.meta.path, &doc_id);
docs.push(ImportDocRef {
entry_index,
doc_id,
title: None,
icon: None,
});
}
}
for (entry_index, entry) in archive.entries.iter().enumerate() {
let lower = entry.meta.path.to_lowercase();
if lower.ends_with(".csv") {
register_csv_path(&mut csv_entries_by_path, &entry.meta.path, entry_index);
} else if !lower.ends_with(".md") {
let blob_id = hash_bytes(&archive.read_entry(entry)?);
blob_ids_by_path.insert(entry.meta.path.clone(), blob_id.clone());
assets.push(ImportAssetRef { entry_index, blob_id });
}
}
if docs.is_empty() {
return Err(ImportError::InvalidSource(
"No Markdown pages found in the archive".to_string(),
));
}
let folders = notion_markdown_folders(&docs, &archive);
let total = docs.len();
let entry_id = docs.first().map(|doc| doc.doc_id.clone());
Ok(Self {
archive,
docs,
assets,
csv_entries_by_path,
page_ids_by_path,
blob_ids_by_path,
folders,
warnings: Vec::new(),
options,
limits,
next_doc: 0,
emitted_assets: BTreeSet::new(),
emitted_final: false,
completed: 0,
total,
entry_id,
})
}
}
impl ImportCursor for NotionMarkdownImportCursor {
fn total(&self) -> usize {
self.total
}
fn next_batch(&mut self) -> ImportResult<Option<ImportBatch>> {
if self.completed >= self.total && self.emitted_final {
return Ok(None);
}
if self.options.cancel_after_entries == Some(self.completed) {
return Err(ImportError::Cancelled);
}
let mut batch = empty_batch(self.total);
batch.entry_id = self.entry_id.clone();
let end = (self.next_doc + self.limits.max_docs.max(1)).min(self.docs.len());
for doc in &self.docs[self.next_doc..end] {
let entry = &self.archive.entries[doc.entry_index];
let bytes = self.archive.read_entry(entry)?;
let source_path = entry.meta.path.clone();
let markdown = String::from_utf8_lossy(&bytes).to_string();
let entry = entry_from_indexed(entry, bytes);
let prepared = prepare_notion_markdown(&entry, &markdown);
let content = rewrite_csv_links(
&prepared.content,
&source_path,
&self.csv_entries_by_path,
&self.archive,
&mut batch.warnings,
)?;
let snapshot = match doc_snapshot(
&doc.doc_id,
&prepared.title,
&content,
&source_path,
&self.page_ids_by_path,
&self.blob_ids_by_path,
) {
Ok(snapshot) => snapshot,
Err(ImportError::Document(error)) => {
push_skipped_doc_warning(&mut batch, &source_path, error);
continue;
}
Err(error) => return Err(error),
};
batch.docs.push(ImportedDocSnapshot {
id: doc.doc_id.clone(),
snapshot,
meta: prepared.meta,
});
}
add_asset_batch(
&self.archive,
&self.limits,
&mut self.emitted_assets,
&mut batch,
&self.assets,
)?;
if end == self.docs.len() && has_emitted_assets(&self.emitted_assets, &self.assets) {
batch.folders.extend(self.folders.clone());
batch.warnings.extend(self.warnings.clone());
self.emitted_final = true;
}
self.next_doc = end;
self.completed = end;
batch.progress = ImportProgress {
completed: self.completed,
total: self.total,
};
batch.done = self.completed >= self.total && self.emitted_final;
Ok(Some(batch))
}
}
fn notion_markdown_folders(docs: &[ImportDocRef], archive: &IndexedArchive) -> Vec<crate::FolderHierarchyDelta> {
let mut folders = Vec::new();
for doc in docs {
let entry = &archive.entries[doc.entry_index];
folders.extend(notion_folders_for_doc_path(&entry.meta.path, &doc.doc_id));
}
folders
}
fn notion_folders_for_doc_path(path: &str, doc_id: &str) -> Vec<crate::FolderHierarchyDelta> {
folder_hierarchy_deltas(notion_folder_parts(path), Some(doc_id), None, strip_notion_hash)
}
fn notion_folder_parts(path: &str) -> Vec<String> {
let mut parts = normalize_import_path(path)
.split('/')
.map(ToString::to_string)
.collect::<Vec<_>>();
let file_name = parts.pop();
if parts.is_empty()
&& let Some(file_name) = file_name
{
parts.push(strip_notion_hash(strip_extension(&file_name)));
}
parts
}
struct PreparedNotionMarkdown {
title: String,
content: String,
meta: Option<ImportedDocMeta>,
}
fn prepare_notion_markdown(entry: &crate::vfs::VfsEntry, markdown: &str) -> PreparedNotionMarkdown {
let (content, frontmatter) = strip_frontmatter(markdown);
let content = strip_notion_reference_footer(&content);
if let Some((title, content)) = content.strip_prefix("# ").and_then(|rest| {
let (title, body) = rest.split_once('\n').unwrap_or((rest, ""));
let title = title.trim();
(!title.is_empty()).then(|| (title.to_string(), body.to_string()))
}) {
return PreparedNotionMarkdown {
meta: frontmatter_meta(&frontmatter, &title),
title,
content,
};
}
let title = frontmatter
.title
.clone()
.unwrap_or_else(|| strip_notion_hash(strip_extension(file_name(&entry.path))));
PreparedNotionMarkdown {
meta: frontmatter_meta(&frontmatter, &title),
title,
content,
}
}
fn strip_notion_reference_footer(markdown: &str) -> String {
let lines = markdown.lines().collect::<Vec<_>>();
let Some(separator) = lines.iter().rposition(|line| line.trim() == "---") else {
return markdown.to_string();
};
let footer = lines[separator + 1..].join("\n");
if !looks_like_notion_reference_footer(&footer) {
return markdown.to_string();
}
lines[..separator].join("\n").trim_end().to_string()
}
fn looks_like_notion_reference_footer(footer: &str) -> bool {
let mut non_empty = 0usize;
let mut link_lines = 0usize;
let mut csv_links = 0usize;
let mut page_links = 0usize;
for line in footer.lines().map(str::trim).filter(|line| !line.is_empty()) {
non_empty += 1;
let Some((_, target, consumed)) = parse_link_span(line) else {
continue;
};
let rest = line[consumed..].trim();
let target = target.to_ascii_lowercase();
if rest.is_empty() {
link_lines += 1;
if target.ends_with(".csv") {
csv_links += 1;
} else if target.ends_with(".md") {
page_links += 1;
}
}
}
non_empty >= 4 && link_lines >= 3 && page_links + csv_links >= 3 && link_lines * 4 >= non_empty * 3
}
fn rewrite_csv_links(
markdown: &str,
current_path: &str,
csv_entries_by_path: &BTreeMap<String, usize>,
archive: &IndexedArchive,
warnings: &mut Vec<ImportWarning>,
) -> ImportResult<String> {
let mut output = Vec::new();
for line in markdown.lines() {
let trimmed = line.trim();
let Some((label, target, consumed)) = parse_link_span(trimmed) else {
output.push(line.to_string());
continue;
};
if consumed != trimmed.len() {
output.push(line.to_string());
continue;
}
let resolved = resolve_path(current_path, &target);
if !resolved.to_ascii_lowercase().ends_with(".csv") {
output.push(line.to_string());
continue;
}
let Some(entry_index) = csv_entries_by_path.get(&resolved) else {
output.push(line.to_string());
continue;
};
let entry = &archive.entries[*entry_index];
let bytes = archive.read_entry(entry)?;
let csv = String::from_utf8_lossy(&bytes);
let table = match csv_to_markdown_table(&csv) {
Some(table) => table,
None => {
warnings.push(ImportWarning {
code: "skipped_csv_table".to_string(),
source_path: Some(entry.meta.path.clone()),
message: format!("Skipped CSV table with no rows: {}", entry.meta.path),
});
continue;
}
};
let label = label.trim();
if !label.is_empty() {
output.push(format!("**{}**", escape_markdown_cell(label)));
output.push(String::new());
}
output.extend(table.lines().map(ToString::to_string));
}
Ok(output.join("\n"))
}
fn csv_to_markdown_table(csv: &str) -> Option<String> {
let mut rows = parse_csv_rows(csv);
rows.retain(|row| row.iter().any(|cell| !cell.trim().is_empty()));
if rows.is_empty() {
return None;
}
let width = rows.iter().map(Vec::len).max().unwrap_or(0).max(1);
for row in &mut rows {
row.resize(width, String::new());
}
let mut table = String::new();
table.push_str(&markdown_table_row(&rows[0]));
table.push('\n');
table.push_str(&markdown_table_row(&vec!["---".to_string(); width]));
for row in rows.iter().skip(1) {
table.push('\n');
table.push_str(&markdown_table_row(row));
}
Some(table)
}
fn markdown_table_row(row: &[String]) -> String {
format!(
"| {} |",
row
.iter()
.map(|cell| escape_markdown_cell(cell))
.collect::<Vec<_>>()
.join(" | ")
)
}
fn escape_markdown_cell(cell: &str) -> String {
cell
.replace('\\', "\\\\")
.replace('|', "\\|")
.replace(['\r', '\n'], " ")
}
fn parse_csv_rows(csv: &str) -> Vec<Vec<String>> {
let mut rows = Vec::new();
let mut row = Vec::new();
let mut cell = String::new();
let mut chars = csv.chars().peekable();
let mut quoted = false;
while let Some(ch) = chars.next() {
match ch {
'"' if quoted && chars.peek() == Some(&'"') => {
cell.push('"');
chars.next();
}
'"' => quoted = !quoted,
',' if !quoted => {
row.push(std::mem::take(&mut cell));
}
'\n' if !quoted => {
row.push(std::mem::take(&mut cell));
rows.push(std::mem::take(&mut row));
}
'\r' if !quoted => {
if chars.peek() == Some(&'\n') {
chars.next();
}
row.push(std::mem::take(&mut cell));
rows.push(std::mem::take(&mut row));
}
_ => cell.push(ch),
}
}
if !cell.is_empty() || !row.is_empty() {
row.push(cell);
rows.push(row);
}
rows
}