#[macro_use]
extern crate lazy_static;
mod walker;
pub use walker::{vault_contents, WalkOptions};
use pathdiff::diff_paths;
use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
use pulldown_cmark::{CodeBlockKind, CowStr, Event, Options, Parser, Tag};
use pulldown_cmark_to_cmark::cmark_with_options;
use rayon::prelude::*;
use regex::Regex;
use slug::slugify;
use snafu::{ResultExt, Snafu};
use std::ffi::OsString;
use std::fmt;
use std::fs::{self, File};
use std::io::prelude::*;
use std::io::ErrorKind;
use std::path::{Path, PathBuf};
use std::str;
type Result<T, E = ExportError> = std::result::Result<T, E>;
type MarkdownTree<'a> = Vec<Event<'a>>;
lazy_static! {
static ref OBSIDIAN_NOTE_LINK_RE: Regex =
Regex::new(r"^(?P<file>[^#|]+)(#(?P<section>.+?))??(\|(?P<label>.+?))??$").unwrap();
}
const PERCENTENCODE_CHARS: &AsciiSet = &CONTROLS.add(b' ').add(b'(').add(b')').add(b'%');
const NOTE_RECURSION_LIMIT: usize = 10;
#[non_exhaustive]
#[derive(Debug, Snafu)]
pub enum ExportError {
#[snafu(display("failed to read from '{}'", path.display()))]
ReadError {
path: PathBuf,
source: std::io::Error,
},
#[snafu(display("failed to write to '{}'", path.display()))]
WriteError {
path: PathBuf,
source: std::io::Error,
},
#[snafu(display("Encountered an error while trying to walk '{}'", path.display()))]
WalkDirError {
path: PathBuf,
source: ignore::Error,
},
#[snafu(display("No such file or directory: {}", path.display()))]
PathDoesNotExist { path: PathBuf },
#[snafu(display("Invalid character encoding encountered"))]
CharacterEncodingError { source: str::Utf8Error },
#[snafu(display("Recursion limit exceeded"))]
RecursionLimitExceeded { file_tree: Vec<PathBuf> },
#[snafu(display("Failed to export '{}'", path.display()))]
FileExportError {
path: PathBuf,
#[snafu(source(from(ExportError, Box::new)))]
source: Box<ExportError>,
},
}
#[derive(Debug, Clone, Copy)]
pub enum FrontmatterStrategy {
Auto,
Always,
Never,
}
#[derive(Debug, Clone)]
pub struct Exporter<'a> {
root: PathBuf,
destination: PathBuf,
frontmatter_strategy: FrontmatterStrategy,
vault_contents: Option<Vec<PathBuf>>,
walk_options: WalkOptions<'a>,
process_embeds_recursively: bool,
}
#[derive(Debug, Clone)]
struct Context {
file_tree: Vec<PathBuf>,
frontmatter_strategy: FrontmatterStrategy,
}
#[derive(Debug, Clone)]
struct ObsidianNoteReference<'a> {
file: &'a str,
section: Option<&'a str>,
label: Option<&'a str>,
}
impl Context {
fn new(file: PathBuf) -> Context {
Context {
file_tree: vec![file.clone()],
frontmatter_strategy: FrontmatterStrategy::Auto,
}
}
fn from_parent(context: &Context, child: &PathBuf) -> Context {
let mut context = context.clone();
context.file_tree.push(child.to_path_buf());
context
}
fn set_frontmatter_strategy(&mut self, strategy: FrontmatterStrategy) -> &mut Context {
self.frontmatter_strategy = strategy;
self
}
fn current_file(&self) -> &PathBuf {
self.file_tree
.last()
.expect("Context not initialized properly, file_tree is empty")
}
fn root_file(&self) -> &PathBuf {
self.file_tree
.first()
.expect("Context not initialized properly, file_tree is empty")
}
fn note_depth(&self) -> usize {
self.file_tree.len()
}
fn file_tree(&self) -> Vec<PathBuf> {
self.file_tree.clone()
}
}
impl<'a> ObsidianNoteReference<'a> {
fn from_str(text: &str) -> ObsidianNoteReference {
let captures = OBSIDIAN_NOTE_LINK_RE
.captures(&text)
.expect("note link regex didn't match - bad input?");
let file = captures
.name("file")
.expect("Obsidian links should always reference a file")
.as_str();
let label = captures.name("label").map(|v| v.as_str());
let section = captures.name("section").map(|v| v.as_str());
ObsidianNoteReference {
file,
label,
section,
}
}
fn display(&self) -> String {
format!("{}", self)
}
}
impl<'a> fmt::Display for ObsidianNoteReference<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let label = self
.label
.map(|v| v.to_string())
.unwrap_or_else(|| match self.section {
Some(section) => format!("{} > {}", self.file, section),
None => self.file.to_string(),
})
.to_string();
write!(f, "{}", label)
}
}
impl<'a> Exporter<'a> {
pub fn new(source: PathBuf, destination: PathBuf) -> Exporter<'a> {
Exporter {
root: source,
destination,
frontmatter_strategy: FrontmatterStrategy::Auto,
walk_options: WalkOptions::default(),
process_embeds_recursively: true,
vault_contents: None,
}
}
pub fn walk_options(&mut self, options: WalkOptions<'a>) -> &mut Exporter<'a> {
self.walk_options = options;
self
}
pub fn frontmatter_strategy(&mut self, strategy: FrontmatterStrategy) -> &mut Exporter<'a> {
self.frontmatter_strategy = strategy;
self
}
pub fn process_embeds_recursively(&mut self, recursive: bool) -> &mut Exporter<'a> {
self.process_embeds_recursively = recursive;
self
}
pub fn run(&mut self) -> Result<()> {
if !self.root.exists() {
return Err(ExportError::PathDoesNotExist {
path: self.root.clone(),
});
}
if self.root.is_file() {
self.vault_contents = Some(vec![self.root.clone()]);
let source_filename = self
.root
.file_name()
.expect("File without a filename? How is that possible?")
.to_string_lossy();
let destination = match self.destination.is_dir() {
true => self.destination.join(String::from(source_filename)),
false => {
let parent = self.destination.parent().unwrap_or(&self.destination);
if !parent.exists() {
return Err(ExportError::PathDoesNotExist {
path: parent.to_path_buf(),
});
}
self.destination.clone()
}
};
return Ok(self.export_note(&self.root, &destination)?);
}
if !self.destination.exists() {
return Err(ExportError::PathDoesNotExist {
path: self.destination.clone(),
});
}
self.vault_contents = Some(vault_contents(
self.root.as_path(),
self.walk_options.clone(),
)?);
self.vault_contents
.as_ref()
.unwrap()
.clone()
.into_par_iter()
.try_for_each(|file| {
let relative_path = file
.strip_prefix(&self.root.clone())
.expect("file should always be nested under root")
.to_path_buf();
let destination = &self.destination.join(&relative_path);
self.export_note(&file, destination)
})?;
Ok(())
}
fn export_note(&self, src: &Path, dest: &Path) -> Result<()> {
match is_markdown_file(src) {
true => self.parse_and_export_obsidian_note(src, dest, self.frontmatter_strategy),
false => copy_file(src, dest),
}
.context(FileExportError { path: src })
}
fn parse_and_export_obsidian_note(
&self,
src: &Path,
dest: &Path,
frontmatter_strategy: FrontmatterStrategy,
) -> Result<()> {
let content = fs::read_to_string(&src).context(ReadError { path: src })?;
let (mut frontmatter, _content) =
matter::matter(&content).unwrap_or(("".to_string(), content.to_string()));
frontmatter = frontmatter.trim().to_string();
let mut outfile = create_file(&dest)?;
let write_frontmatter = match frontmatter_strategy {
FrontmatterStrategy::Always => true,
FrontmatterStrategy::Never => false,
FrontmatterStrategy::Auto => frontmatter != "",
};
if write_frontmatter {
if frontmatter != "" && !frontmatter.ends_with('\n') {
frontmatter.push('\n');
}
outfile
.write_all(format!("---\n{}---\n\n", frontmatter).as_bytes())
.context(WriteError { path: &dest })?;
}
let mut context = Context::new(src.to_path_buf());
context.set_frontmatter_strategy(frontmatter_strategy);
let markdown_tree = self.parse_obsidian_note(&src, &context)?;
outfile
.write_all(render_mdtree_to_mdtext(markdown_tree).as_bytes())
.context(WriteError { path: &dest })?;
Ok(())
}
fn parse_obsidian_note<'b>(&self, path: &Path, context: &Context) -> Result<MarkdownTree<'b>> {
if context.note_depth() > NOTE_RECURSION_LIMIT {
return Err(ExportError::RecursionLimitExceeded {
file_tree: context.file_tree(),
});
}
let content = fs::read_to_string(&path).context(ReadError { path })?;
let (_frontmatter, content) =
matter::matter(&content).unwrap_or(("".to_string(), content.to_string()));
let mut parser_options = Options::empty();
parser_options.insert(Options::ENABLE_TABLES);
parser_options.insert(Options::ENABLE_FOOTNOTES);
parser_options.insert(Options::ENABLE_STRIKETHROUGH);
parser_options.insert(Options::ENABLE_TASKLISTS);
let mut parser = Parser::new_ext(&content, parser_options);
let mut tree = vec![];
let mut buffer = Vec::with_capacity(5);
while let Some(event) = parser.next() {
match event {
Event::Text(CowStr::Borrowed("[")) | Event::Text(CowStr::Borrowed("![")) => {
buffer.push(event);
for _ in 1..5 {
if let Some(event) = parser.next() {
buffer.push(event);
}
}
if buffer.len() != 5
|| buffer[1] != Event::Text(CowStr::Borrowed("["))
|| buffer[3] != Event::Text(CowStr::Borrowed("]"))
|| buffer[4] != Event::Text(CowStr::Borrowed("]"))
{
tree.append(&mut buffer);
buffer.clear();
continue;
}
if let Event::Text(CowStr::Borrowed(text)) = buffer[2] {
match buffer[0] {
Event::Text(CowStr::Borrowed("[")) => {
let mut elements = self.make_link_to_file(
ObsidianNoteReference::from_str(&text),
context,
);
tree.append(&mut elements);
buffer.clear();
continue;
}
Event::Text(CowStr::Borrowed("![")) => {
let mut elements = self.embed_file(&text, &context)?;
tree.append(&mut elements);
buffer.clear();
continue;
}
_ => {}
}
}
}
_ => tree.push(event),
}
if !buffer.is_empty() {
tree.append(&mut buffer);
buffer.clear();
}
}
tree.append(&mut buffer);
Ok(tree.into_iter().map(event_to_owned).collect())
}
fn embed_file<'b>(&self, link_text: &'a str, context: &'a Context) -> Result<MarkdownTree<'a>> {
let note_ref = ObsidianNoteReference::from_str(link_text);
let path = lookup_filename_in_vault(note_ref.file, &self.vault_contents.as_ref().unwrap());
if path.is_none() {
eprintln!(
"Warning: Unable to find embedded note\n\tReference: '{}'\n\tSource: '{}'\n",
note_ref.file,
context.current_file().display(),
);
return Ok(vec![]);
}
let path = path.unwrap();
let child_context = Context::from_parent(context, path);
let no_ext = OsString::new();
if !self.process_embeds_recursively && context.file_tree.contains(path) {
return Ok([
vec![Event::Text(CowStr::Borrowed("→ "))],
self.make_link_to_file(note_ref, &child_context),
]
.concat());
}
let tree = match path.extension().unwrap_or(&no_ext).to_str() {
Some("md") => {
let mut tree = self.parse_obsidian_note(&path, &child_context)?;
if let Some(section) = note_ref.section {
tree = reduce_to_section(tree, section);
}
tree
}
Some("png") | Some("jpg") | Some("jpeg") | Some("gif") | Some("webp") => {
self.make_link_to_file(note_ref, &child_context)
.into_iter()
.map(|event| match event {
Event::Start(Tag::Link(linktype, cowstr1, cowstr2)) => {
Event::Start(Tag::Image(
linktype,
CowStr::from(cowstr1.into_string()),
CowStr::from(cowstr2.into_string()),
))
}
Event::End(Tag::Link(linktype, cowstr1, cowstr2)) => {
Event::End(Tag::Image(
linktype,
CowStr::from(cowstr1.into_string()),
CowStr::from(cowstr2.into_string()),
))
}
_ => event,
})
.collect()
}
_ => self.make_link_to_file(note_ref, &child_context),
};
Ok(tree)
}
fn make_link_to_file<'b>(
&self,
reference: ObsidianNoteReference<'b>,
context: &Context,
) -> MarkdownTree<'b> {
let target_file =
lookup_filename_in_vault(reference.file, &self.vault_contents.as_ref().unwrap());
if target_file.is_none() {
eprintln!(
"Warning: Unable to find referenced note\n\tReference: '{}'\n\tSource: '{}'\n",
reference.file,
context.current_file().display(),
);
return vec![
Event::Start(Tag::Emphasis),
Event::Text(CowStr::from(reference.display())),
Event::End(Tag::Emphasis),
];
}
let target_file = target_file.unwrap();
let rel_link = diff_paths(
target_file,
&context
.root_file()
.parent()
.expect("obsidian content files should always have a parent"),
)
.expect("should be able to build relative path when target file is found in vault");
let rel_link = rel_link.to_string_lossy();
let mut link = utf8_percent_encode(&rel_link, PERCENTENCODE_CHARS).to_string();
if let Some(section) = reference.section {
link.push('#');
link.push_str(&slugify(section));
}
let link_tag = pulldown_cmark::Tag::Link(
pulldown_cmark::LinkType::Inline,
CowStr::from(link.to_string()),
CowStr::from(""),
);
vec![
Event::Start(link_tag.clone()),
Event::Text(CowStr::from(reference.display())),
Event::End(link_tag.clone()),
]
}
}
fn lookup_filename_in_vault<'a>(
filename: &str,
vault_contents: &'a [PathBuf],
) -> Option<&'a PathBuf> {
vault_contents.iter().find(|path| {
let path_lowered = PathBuf::from(path.to_string_lossy().to_lowercase());
path.ends_with(&filename)
|| path_lowered.ends_with(&filename.to_lowercase())
|| path.ends_with(format!("{}.md", &filename))
|| path_lowered.ends_with(format!("{}.md", &filename.to_lowercase()))
})
}
fn render_mdtree_to_mdtext(markdown: MarkdownTree) -> String {
let mut buffer = String::new();
cmark_with_options(
markdown.iter(),
&mut buffer,
None,
pulldown_cmark_to_cmark::Options::default(),
)
.expect("formatting to string not expected to fail");
buffer.push('\n');
buffer
}
fn create_file(dest: &Path) -> Result<File> {
let file = File::create(&dest)
.or_else(|err| {
if err.kind() == ErrorKind::NotFound {
let parent = dest.parent().expect("file should have a parent directory");
if let Err(err) = std::fs::create_dir_all(&parent) {
return Err(err);
}
}
File::create(&dest)
})
.context(WriteError { path: dest })?;
Ok(file)
}
fn copy_file(src: &Path, dest: &Path) -> Result<()> {
std::fs::copy(&src, &dest)
.or_else(|err| {
if err.kind() == ErrorKind::NotFound {
let parent = dest.parent().expect("file should have a parent directory");
if let Err(err) = std::fs::create_dir_all(&parent) {
return Err(err);
}
}
std::fs::copy(&src, &dest)
})
.context(WriteError { path: dest })?;
Ok(())
}
fn is_markdown_file(file: &Path) -> bool {
let no_ext = OsString::new();
let ext = file.extension().unwrap_or(&no_ext).to_string_lossy();
ext == "md"
}
fn reduce_to_section<'a, 'b>(tree: MarkdownTree<'a>, section: &'b str) -> MarkdownTree<'a> {
let mut new_tree = Vec::with_capacity(tree.len());
let mut target_section_encountered = false;
let mut currently_in_target_section = false;
let mut section_level = 0;
let mut last_level = 0;
let mut last_tag_was_heading = false;
for event in tree.into_iter() {
new_tree.push(event.clone());
match event {
Event::Start(Tag::Heading(level)) => {
last_tag_was_heading = true;
last_level = level;
if currently_in_target_section && level <= section_level {
currently_in_target_section = false;
new_tree.pop();
}
}
Event::Text(cowstr) => {
if !last_tag_was_heading {
last_tag_was_heading = false;
continue;
}
last_tag_was_heading = false;
if cowstr.to_string().to_lowercase() == section.to_lowercase() {
target_section_encountered = true;
currently_in_target_section = true;
section_level = last_level;
let current_event = new_tree.pop().unwrap();
let heading_start_event = new_tree.pop().unwrap();
new_tree.clear();
new_tree.push(heading_start_event);
new_tree.push(current_event);
}
}
_ => {}
}
if target_section_encountered && !currently_in_target_section {
return new_tree;
}
}
new_tree
}
fn event_to_owned<'a>(event: Event) -> Event<'a> {
match event {
Event::Start(tag) => Event::Start(tag_to_owned(tag)),
Event::End(tag) => Event::End(tag_to_owned(tag)),
Event::Text(cowstr) => Event::Text(CowStr::from(cowstr.into_string())),
Event::Code(cowstr) => Event::Code(CowStr::from(cowstr.into_string())),
Event::Html(cowstr) => Event::Html(CowStr::from(cowstr.into_string())),
Event::FootnoteReference(cowstr) => {
Event::FootnoteReference(CowStr::from(cowstr.into_string()))
}
Event::SoftBreak => Event::SoftBreak,
Event::HardBreak => Event::HardBreak,
Event::Rule => Event::Rule,
Event::TaskListMarker(checked) => Event::TaskListMarker(checked),
}
}
fn tag_to_owned<'a>(tag: Tag) -> Tag<'a> {
match tag {
Tag::Paragraph => Tag::Paragraph,
Tag::Heading(level) => Tag::Heading(level),
Tag::BlockQuote => Tag::BlockQuote,
Tag::CodeBlock(codeblock_kind) => Tag::CodeBlock(codeblock_kind_to_owned(codeblock_kind)),
Tag::List(optional) => Tag::List(optional),
Tag::Item => Tag::Item,
Tag::FootnoteDefinition(cowstr) => {
Tag::FootnoteDefinition(CowStr::from(cowstr.into_string()))
}
Tag::Table(alignment_vector) => Tag::Table(alignment_vector),
Tag::TableHead => Tag::TableHead,
Tag::TableRow => Tag::TableRow,
Tag::TableCell => Tag::TableCell,
Tag::Emphasis => Tag::Emphasis,
Tag::Strong => Tag::Strong,
Tag::Strikethrough => Tag::Strikethrough,
Tag::Link(linktype, cowstr1, cowstr2) => Tag::Link(
linktype,
CowStr::from(cowstr1.into_string()),
CowStr::from(cowstr2.into_string()),
),
Tag::Image(linktype, cowstr1, cowstr2) => Tag::Image(
linktype,
CowStr::from(cowstr1.into_string()),
CowStr::from(cowstr2.into_string()),
),
}
}
fn codeblock_kind_to_owned<'a>(codeblock_kind: CodeBlockKind) -> CodeBlockKind<'a> {
match codeblock_kind {
CodeBlockKind::Indented => CodeBlockKind::Indented,
CodeBlockKind::Fenced(cowstr) => CodeBlockKind::Fenced(CowStr::from(cowstr.into_string())),
}
}