use crate::datatypes::values::Value;
use crate::graph::storage::GraphRead;
use crate::graph::DirGraph;
use crate::okf::model::{
Profile, ATTACHMENT_LABEL, DEFAULT_BODY_PROPERTY, FOLDER_LABEL, IMAGE_LABEL, SOURCE_LABEL,
TAG_LABEL,
};
use crate::okf::vault_config::{CONFIG_DIR, RECIPES_DIR, SKILLS_DIR};
use petgraph::graph::NodeIndex;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::path::{Path, PathBuf};
mod manifest;
mod paths;
mod yaml_out;
pub use manifest::{MANIFEST_FILE, MANIFEST_VERSION};
use manifest::Writer;
use paths::{assign_paths, sanitize_segment};
use yaml_out::{lower_snake, render_frontmatter, Tree};
const NEVER_IN_FRONTMATTER: [&str; 4] = ["concept_id", "title", "file_path", "_provisional"];
const SYNTHESIZED_LABELS: [&str; 5] = [
TAG_LABEL,
SOURCE_LABEL,
FOLDER_LABEL,
IMAGE_LABEL,
ATTACHMENT_LABEL,
];
#[derive(Debug, Clone)]
pub struct ExportOptions {
pub force: bool,
pub source_root: Option<PathBuf>,
pub body_property: String,
}
impl Default for ExportOptions {
fn default() -> Self {
ExportOptions {
force: false,
source_root: None,
body_property: DEFAULT_BODY_PROPERTY.to_string(),
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ExportReport {
pub files_written: usize,
pub files_unchanged: usize,
pub files_deleted: usize,
pub files_refused: usize,
pub refusals: Vec<String>,
pub edge_properties_dropped: usize,
pub attachments_copied: usize,
pub attachments_unresolved: usize,
pub skills_written: usize,
pub recipes_written: usize,
}
impl ExportReport {
pub fn render(&self) -> String {
let mut out = String::new();
out.push_str(&format!("files written: {}\n", self.files_written));
out.push_str(&format!("files unchanged: {}\n", self.files_unchanged));
out.push_str(&format!("files deleted: {}\n", self.files_deleted));
out.push_str(&format!("files refused: {}\n", self.files_refused));
out.push_str(&format!(
"attachments copied: {} (unresolved: {})\n",
self.attachments_copied, self.attachments_unresolved
));
out.push_str(&format!(
"skills: {}, recipes: {}\n",
self.skills_written, self.recipes_written
));
out.push_str(&format!(
"edge properties dropped: {}\n",
self.edge_properties_dropped
));
if self.refusals.is_empty() {
out.push_str("refusals: none\n");
} else {
out.push_str("refusals:\n");
for line in &self.refusals {
out.push_str(&format!(" - {line}\n"));
}
}
out
}
}
pub(super) struct Note {
idx: NodeIndex,
pub(super) label: String,
id: String,
title: String,
pub(super) file_path: Option<String>,
props: Vec<(String, Value)>,
body: Option<String>,
pub(super) out: String,
}
impl Note {
fn stem(&self) -> &str {
let file = self.out.rsplit('/').next().unwrap_or(&self.out);
file.strip_suffix(".md").unwrap_or(file)
}
fn qualified(&self) -> &str {
self.out.strip_suffix(".md").unwrap_or(&self.out)
}
pub(super) fn display_name(&self) -> &str {
if self.title.is_empty() {
&self.id
} else {
&self.title
}
}
pub(super) fn id(&self) -> &str {
&self.id
}
}
pub fn export(graph: &DirGraph, dir: &Path, opts: &ExportOptions) -> Result<ExportReport, String> {
if dir.exists() && !dir.is_dir() {
return Err(format!("{} is not a directory", dir.display()));
}
std::fs::create_dir_all(dir).map_err(|e| format!("creating {}: {e}", dir.display()))?;
let Collected {
mut notes,
attachments,
stubs,
} = collect(graph, &opts.body_property);
assign_paths(&mut notes);
let index = LinkIndex::new(¬es);
let (edges, edge_properties_dropped) = outgoing_edges(graph, ¬es, &stubs);
let mut writer = Writer::open(dir, opts.force)?;
writer.report.edge_properties_dropped = edge_properties_dropped;
for note in ¬es {
let text = render_note(note, ¬es, &index, &edges);
writer.put(¬e.out.clone(), text.as_bytes())?;
}
write_carried(graph, &mut writer)?;
let source_root = opts
.source_root
.clone()
.or_else(|| graph.source_root.as_ref().map(PathBuf::from));
copy_attachments(&attachments, source_root.as_deref(), &mut writer)?;
writer.finish()
}
fn collect(graph: &DirGraph, body_property: &str) -> Collected {
let _arena_guard = graph.graph.begin_query();
let skill_label = crate::graph::skills::SKILL_LABEL;
let recipe_label = crate::graph::recipes::RECIPE_LABEL;
let mut attachments = Vec::new();
let mut stubs: HashMap<NodeIndex, String> = HashMap::new();
let mut vault_built = false;
let mut candidates: Vec<Note> = Vec::new();
for idx in graph.graph.node_indices() {
let Some(view) = graph.node_view(idx) else {
continue;
};
let label = view.node_type_str(&graph.interner).to_string();
let provisional = matches!(
view.get_property_value("_provisional"),
Some(Value::Boolean(true))
);
if label == IMAGE_LABEL || label == ATTACHMENT_LABEL {
if !provisional {
if let Value::String(path) = view.id().as_ref() {
attachments.push(path.clone());
}
}
continue;
}
if provisional {
if let Value::String(name) = view.id().as_ref() {
stubs.insert(idx, name.clone());
}
continue;
}
if SYNTHESIZED_LABELS.contains(&label.as_str())
|| label == skill_label
|| label == recipe_label
{
continue;
}
let file_path = match view.get_property_value("file_path") {
Some(Value::String(p)) if !p.is_empty() => Some(p),
_ => None,
};
if file_path.is_some() {
vault_built = true;
}
let mut props = view.property_pairs_named(&graph.interner);
props.retain(|(k, v)| {
!NEVER_IN_FRONTMATTER.contains(&k.as_str()) && !matches!(v, Value::Null)
});
props.sort_by(|a, b| a.0.cmp(&b.0));
let body = props
.iter()
.position(|(k, _)| k == body_property)
.map(|at| props.remove(at).1)
.map(|v| match v {
Value::String(s) => s,
other => crate::datatypes::values::raw_string(&other),
});
candidates.push(Note {
idx,
label,
id: scalar_string(&view.id()),
title: scalar_string(&view.title()),
file_path,
props,
body,
out: String::new(),
});
}
let mut notes: Vec<Note> = candidates
.into_iter()
.filter(|note| !vault_built || note.file_path.is_some())
.collect();
notes.sort_by(|a, b| a.label.cmp(&b.label).then_with(|| a.id.cmp(&b.id)));
attachments.sort();
attachments.dedup();
Collected {
notes,
attachments,
stubs,
}
}
struct Collected {
notes: Vec<Note>,
attachments: Vec<String>,
stubs: HashMap<NodeIndex, String>,
}
fn scalar_string(value: &Value) -> String {
match value {
Value::String(s) => s.clone(),
Value::Null => String::new(),
other => crate::datatypes::values::raw_string(other),
}
}
struct LinkIndex {
stem_uses: HashMap<String, usize>,
}
impl LinkIndex {
fn new(notes: &[Note]) -> Self {
let mut stem_uses: HashMap<String, usize> = HashMap::new();
for note in notes {
*stem_uses
.entry(note.stem().to_ascii_lowercase())
.or_default() += 1;
}
LinkIndex { stem_uses }
}
fn wikilink(&self, note: &Note) -> String {
if self.stem_uses.get(¬e.stem().to_ascii_lowercase()) > Some(&1) {
note.qualified().to_string()
} else {
note.stem().to_string()
}
}
}
struct OutEdge {
conn_type: String,
target: Target,
}
enum Target {
Note(usize),
Stub(String),
}
fn outgoing_edges(
graph: &DirGraph,
notes: &[Note],
stubs: &HashMap<NodeIndex, String>,
) -> (HashMap<NodeIndex, Vec<OutEdge>>, usize) {
let positions: HashMap<NodeIndex, usize> = notes
.iter()
.enumerate()
.map(|(at, note)| (note.idx, at))
.collect();
let mut out: HashMap<NodeIndex, Vec<OutEdge>> = HashMap::new();
let mut dropped = 0usize;
for edge in graph.graph.edge_indices() {
let Some((src, tgt)) = graph.graph.edge_endpoints(edge) else {
continue;
};
if !positions.contains_key(&src) {
continue;
}
let Some(data) = graph.graph.edge_weight(edge) else {
continue;
};
dropped += data.properties.len();
let target = match positions.get(&tgt) {
Some(&at) => Target::Note(at),
None => match stubs.get(&tgt) {
Some(name) => Target::Stub(name.clone()),
None => continue,
},
};
let conn_type = data.connection_type_str(&graph.interner).to_string();
out.entry(src)
.or_default()
.push(OutEdge { conn_type, target });
}
(out, dropped)
}
fn render_note(
note: &Note,
notes: &[Note],
index: &LinkIndex,
edges: &HashMap<NodeIndex, Vec<OutEdge>>,
) -> String {
let mut tree = Tree::default();
if note.id != note.stem() {
tree.insert("id", Value::String(note.id.clone()));
}
if !note.title.is_empty() && note.title != recovered_title(note) {
tree.insert("title", Value::String(note.title.clone()));
}
for (key, value) in ¬e.props {
tree.insert(key, value.clone());
}
for (key, targets) in edge_keys(note, notes, index, edges) {
tree.insert_wikilinks(&key, targets);
}
let front = render_frontmatter(&tree);
let body = note.body.as_deref().unwrap_or("");
match (front.is_empty(), body.is_empty()) {
(true, true) => String::new(),
(true, false) => ensure_newline(body),
(false, true) => format!("---\n{front}---\n"),
(false, false) => format!("---\n{front}---\n{}", ensure_newline(body)),
}
}
fn ensure_newline(text: &str) -> String {
if text.ends_with('\n') {
text.to_string()
} else {
format!("{text}\n")
}
}
fn edge_keys(
note: &Note,
notes: &[Note],
index: &LinkIndex,
edges: &HashMap<NodeIndex, Vec<OutEdge>>,
) -> BTreeMap<String, Vec<String>> {
let Some(outgoing) = edges.get(¬e.idx) else {
return BTreeMap::new();
};
let mentioned = body_links(note);
let mut by_key: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
for edge in outgoing {
let name = match &edge.target {
Target::Note(at) => {
let target = ¬es[*at];
if body_states(&mentioned, &edge.conn_type, |name| names(target, name)) {
continue;
}
index.wikilink(target)
}
Target::Stub(name) => {
let lowered = name.to_ascii_lowercase();
if body_states(&mentioned, &edge.conn_type, |written| written == lowered) {
continue;
}
name.clone()
}
};
by_key
.entry(lower_snake(&edge.conn_type))
.or_default()
.insert(name);
}
by_key
.into_iter()
.map(|(key, targets)| (key, targets.into_iter().collect()))
.collect()
}
fn recovered_title(note: &Note) -> String {
let named = note
.props
.iter()
.find(|(k, _)| k == "name")
.and_then(|(_, v)| match v {
Value::String(s) if !s.is_empty() => Some(s.clone()),
Value::Null | Value::String(_) => None,
other => Some(crate::datatypes::values::raw_string(other)),
});
named
.or_else(|| note.body.as_deref().and_then(crate::okf::first_heading))
.unwrap_or_else(|| note.stem().to_string())
}
fn body_links(note: &Note) -> Vec<(String, String)> {
let Some(body) = note.body.as_deref() else {
return Vec::new();
};
let dir = match note.out.rfind('/') {
Some(at) => ¬e.out[..at],
None => "",
};
crate::okf::links::extract(body, dir, &Profile::obsidian())
.links
.into_iter()
.map(|link| {
(
link.conn_type,
link.target.trim_end_matches(".md").to_ascii_lowercase(),
)
})
.collect()
}
fn body_states(
written: &[(String, String)],
conn_type: &str,
names_target: impl Fn(&str) -> bool,
) -> bool {
written
.iter()
.any(|(conn, target)| conn == conn_type && names_target(target))
}
fn names(note: &Note, target: &str) -> bool {
if target == note.stem().to_ascii_lowercase()
|| target == note.id.to_ascii_lowercase()
|| target == note.qualified().to_ascii_lowercase()
|| (!note.title.is_empty() && target == note.title.to_ascii_lowercase())
{
return true;
}
note.props
.iter()
.find(|(k, _)| k == "aliases")
.is_some_and(|(_, v)| match v {
Value::List(items) => items.iter().any(|item| match item {
Value::String(s) => s.to_ascii_lowercase() == target,
_ => false,
}),
_ => false,
})
}
fn write_carried(graph: &DirGraph, writer: &mut Writer) -> Result<(), String> {
for summary in crate::graph::skills::list(graph) {
let record = crate::graph::skills::get(graph, &summary.name)
.map_err(|e| format!("reading skill {}: {e}", summary.name))?;
let path = format!(
"{CONFIG_DIR}/{SKILLS_DIR}/{}.md",
sanitize_segment(&record.name)
);
let text = crate::graph::skills::render_markdown(&record);
writer.put(&path, text.as_bytes())?;
writer.report.skills_written += 1;
}
for record in crate::graph::recipes::list(graph) {
let path = format!(
"{CONFIG_DIR}/{RECIPES_DIR}/{}.{}.md",
sanitize_segment(&record.recipe),
sanitize_segment(&record.name)
);
let text = crate::graph::recipes::render_markdown(&record);
writer.put(&path, text.as_bytes())?;
writer.report.recipes_written += 1;
}
Ok(())
}
fn copy_attachments(
attachments: &[String],
root: Option<&Path>,
writer: &mut Writer,
) -> Result<(), String> {
let Some(root) = root else {
writer.report.attachments_unresolved += attachments.len();
return Ok(());
};
for rel in attachments {
let from = root.join(rel);
match std::fs::read(&from) {
Ok(bytes) => {
let modified = std::fs::metadata(&from)
.ok()
.and_then(|m| m.modified().ok());
writer.put_at(rel, &bytes, modified)?;
writer.report.attachments_copied += 1;
}
Err(_) => writer.report.attachments_unresolved += 1,
}
}
Ok(())
}
#[cfg(test)]
mod export_tests;
#[cfg(test)]
mod roundtrip_tests;