use std::collections::{BTreeMap, BTreeSet};
use nanoid::nanoid;
use serde_json::Value as JsonValue;
use super::{
IndexedArchive, doc_snapshot, empty_batch, file_name, folder_hierarchy_deltas, hash_bytes, mime_from_path,
parse_date_millis, read_archive, strip_extension,
};
use crate::{
FolderHierarchyDelta, ImportBatch, ImportBatchLimits, ImportCursor, ImportError, ImportOptions, ImportProgress,
ImportProvider, ImportResult, ImportSource, ImportedAsset, ImportedDocMeta, ImportedDocSnapshot, ImportedTag,
};
fn split_textbundle_path(path: &str) -> Option<(&str, &str)> {
let marker = ".textbundle/";
let index = path.to_lowercase().find(marker)?;
let split = index + ".textbundle".len();
Some((&path[..split], &path[split + 1..]))
}
pub(super) struct BearZipProvider;
impl ImportProvider for BearZipProvider {
fn format(&self) -> &'static str {
"bearZip"
}
fn create_cursor(
&self,
source: ImportSource,
options: ImportOptions,
limits: ImportBatchLimits,
) -> ImportResult<Box<dyn ImportCursor>> {
let archive = read_archive(source, false, &options)?;
Ok(Box::new(BearImportCursor::new(archive, options, limits)?))
}
}
struct BearBundleRef {
bundle_path: String,
doc_id: String,
markdown: usize,
info: Option<usize>,
assets: Vec<usize>,
}
struct BearImportCursor {
archive: IndexedArchive,
bundles: Vec<BearBundleRef>,
options: ImportOptions,
limits: ImportBatchLimits,
next_doc: usize,
emitted_final: bool,
completed: usize,
total: usize,
entry_id: Option<String>,
}
impl BearImportCursor {
fn new(archive: IndexedArchive, options: ImportOptions, limits: ImportBatchLimits) -> ImportResult<Self> {
let mut bundles = BTreeMap::<String, (Option<usize>, Option<usize>, Vec<usize>)>::new();
for (entry_index, entry) in archive.entries.iter().enumerate() {
let Some((bundle_path, inner_path)) = split_textbundle_path(&entry.meta.path) else {
continue;
};
let bundle = bundles.entry(bundle_path.to_string()).or_default();
match inner_path {
"text.md" | "text.txt" => bundle.0 = Some(entry_index),
"info.json" => bundle.1 = Some(entry_index),
path if path.starts_with("assets/") => bundle.2.push(entry_index),
_ => {}
}
}
let mut bundle_refs = Vec::new();
for (bundle_path, (markdown, info, assets)) in bundles {
if let Some(markdown) = markdown {
bundle_refs.push(BearBundleRef {
bundle_path,
doc_id: nanoid!(),
markdown,
info,
assets,
});
}
}
if bundle_refs.is_empty() {
return Err(ImportError::InvalidSource(
"No valid Bear textbundles found in the archive".to_string(),
));
}
let total = bundle_refs.len();
let entry_id = bundle_refs.first().map(|bundle| bundle.doc_id.clone());
Ok(Self {
archive,
bundles: bundle_refs,
options,
limits,
next_doc: 0,
emitted_final: false,
completed: 0,
total,
entry_id,
})
}
fn plan_bundle(
&self,
batch: &mut ImportBatch,
bundle: &BearBundleRef,
tags: &mut BTreeMap<String, Vec<String>>,
) -> ImportResult<()> {
let markdown_entry = &self.archive.entries[bundle.markdown];
let markdown_bytes = self.archive.read_entry(markdown_entry)?;
let info = if let Some(info_index) = bundle.info {
let info_entry = &self.archive.entries[info_index];
serde_json::from_slice::<JsonValue>(&self.archive.read_entry(info_entry)?).ok()
} else {
None
};
if info
.as_ref()
.and_then(|value| value.pointer("/net.shinyfrog.bear/trashed"))
.and_then(JsonValue::as_i64)
== Some(1)
{
return Ok(());
}
let raw_markdown = String::from_utf8_lossy(&markdown_bytes);
if raw_markdown.trim().is_empty() {
return Ok(());
}
let (tag_names, content) = parse_bear_tags(&raw_markdown);
for tag in &tag_names {
tags.entry(tag.clone()).or_default().push(bundle.doc_id.clone());
}
let title = first_h1(&content).unwrap_or_else(|| {
strip_extension(file_name(bundle.bundle_path.trim_end_matches('/')))
.trim_end_matches(".textbundle")
.to_string()
});
let mut blob_ids_by_path = BTreeMap::new();
for asset_index in &bundle.assets {
let entry = &self.archive.entries[*asset_index];
let bytes = self.archive.read_entry(entry)?;
let blob_id = hash_bytes(&bytes);
let imported = ImportedAsset {
blob_id: blob_id.clone(),
source_path: entry.meta.path.clone(),
file_name: file_name(&entry.meta.path).to_string(),
mime: mime_from_path(&entry.meta.path).to_string(),
bytes,
};
blob_ids_by_path.insert(entry.meta.path.clone(), blob_id.clone());
if let Some((_, relative)) = split_textbundle_path(&entry.meta.path) {
blob_ids_by_path.insert(relative.to_string(), blob_id);
}
batch.blobs.push(imported);
}
let content = convert_bear_markdown(&content);
let snapshot = doc_snapshot(
&bundle.doc_id,
&title,
&content,
&markdown_entry.meta.path,
&BTreeMap::new(),
&blob_ids_by_path,
)?;
batch.docs.push(ImportedDocSnapshot {
id: bundle.doc_id.clone(),
snapshot,
meta: Some(ImportedDocMeta {
title: Some(title),
create_date: bear_date(&info, "creationDate"),
updated_date: bear_date(&info, "modificationDate"),
tags: (!tag_names.is_empty()).then_some(tag_names),
favorite: None,
trash: None,
}),
});
batch.folders.extend(bear_folders(&bundle.doc_id, tags));
Ok(())
}
}
impl ImportCursor for BearImportCursor {
fn total(&self) -> usize {
self.total
}
fn next_batch(&mut self) -> ImportResult<Option<ImportBatch>> {
if self.completed >= self.total && self.emitted_final {
return Ok(None);
}
if self.options.cancel_after_entries == Some(self.completed) {
return Err(ImportError::Cancelled);
}
let mut batch = empty_batch(self.total);
batch.entry_id = self.entry_id.clone();
let end = (self.next_doc + self.limits.max_docs.max(1)).min(self.bundles.len());
let mut tags = BTreeMap::<String, Vec<String>>::new();
for bundle in &self.bundles[self.next_doc..end] {
self.plan_bundle(&mut batch, bundle, &mut tags)?;
}
if end == self.bundles.len() {
batch.tags = tags
.into_iter()
.map(|(name, doc_ids)| ImportedTag { name, doc_ids })
.collect();
self.emitted_final = true;
}
self.next_doc = end;
self.completed = end;
batch.progress = ImportProgress {
completed: self.completed,
total: self.total,
};
batch.done = self.completed >= self.total && self.emitted_final;
Ok(Some(batch))
}
}
fn parse_bear_tags(markdown: &str) -> (Vec<String>, String) {
let mut tags = BTreeSet::new();
let mut lines = Vec::new();
for line in markdown.lines() {
let trimmed = line.trim();
if let Some(tag) = trimmed
.strip_prefix('#')
.filter(|tag| !tag.starts_with(' ') && !tag.is_empty())
.map(|tag| tag.trim_end_matches('#').trim())
.filter(|tag| !tag.is_empty())
{
tags.insert(tag.to_string());
continue;
}
lines.push(line);
}
(tags.into_iter().collect(), lines.join("\n"))
}
fn first_h1(markdown: &str) -> Option<String> {
markdown.lines().find_map(|line| {
line
.strip_prefix("# ")
.map(str::trim)
.filter(|title| !title.is_empty())
.map(ToString::to_string)
})
}
fn convert_bear_markdown(markdown: &str) -> String {
markdown.replace("==", "")
}
fn bear_date(info: &Option<JsonValue>, key: &str) -> Option<i64> {
info
.as_ref()
.and_then(|value| value.pointer(&format!("/net.shinyfrog.bear/{key}")))
.and_then(JsonValue::as_str)
.and_then(parse_date_millis)
}
fn bear_folders(doc_id: &str, tags: &BTreeMap<String, Vec<String>>) -> Vec<FolderHierarchyDelta> {
let mut folders = Vec::new();
for tag in tags.keys() {
folders.extend(folder_hierarchy_deltas(
tag.split('/').filter(|part| !part.is_empty()),
Some(doc_id),
None,
|part| part.to_string(),
));
}
folders
}
#[cfg(test)]
mod tests {
use super::{
super::tests::{plan_zip, zip},
*,
};
fn plan_bear_zip(bytes: impl AsRef<[u8]>) -> ImportResult<ImportBatch> {
plan_zip("bearZip", bytes)
}
#[test]
fn bear_zip_plans_textbundle_meta_tags_and_assets() {
let bytes = zip(&[
(
"Notes/Idea.textbundle/text.md",
b"# Bear Title\nbody\n\n\n\n==green highlight==\n\n#work/project\n#Blue Tag#",
),
(
"Notes/Idea.textbundle/info.json",
br#"{"net.shinyfrog.bear":{"creationDate":"2018-04-12T09:51:00.000Z","modificationDate":"2018-04-12T10:00:00.000Z"}}"#,
),
("Notes/Idea.textbundle/assets/photo.png", &[137, 80, 78, 71]),
]);
let batch = plan_bear_zip(bytes).unwrap();
let snapshot = serde_json::to_string(&batch.docs[0].snapshot).unwrap();
assert_eq!(batch.docs[0].snapshot["meta"]["title"], "Bear Title");
assert_eq!(batch.docs[0].meta.as_ref().unwrap().create_date, Some(1523526660000));
assert_eq!(
batch.tags.iter().map(|tag| tag.name.as_str()).collect::<Vec<_>>(),
["Blue Tag", "work/project"]
);
assert!(snapshot.contains("\"sourceId\""));
assert!(snapshot.contains("green highlight"));
}
}