specfold_core/import/
plan.rs

1//! Builds the import plan: one row per source for review, and the file changes that apply it.
2
3use super::convert;
4use super::key;
5use super::map::ImportMap;
6use super::owner;
7use super::sources::{self, Unit};
8use crate::authoring::Plan;
9use crate::corpus::Corpus;
10use crate::files::hash;
11use crate::markdown::{body, frontmatter};
12use crate::model::doctype::{Form, KeyKind};
13use crate::model::{DocId, Tier, stamp};
14use crate::placement::paths;
15use anyhow::{Result, anyhow};
16use regex::Regex;
17use serde::Serialize;
18use serde_json::{Map, Value, json};
19use std::collections::{BTreeMap, BTreeSet};
20
21#[derive(Debug, Clone, Serialize)]
22pub struct Row {
23    pub source: String,
24    /// `doc` or `redirect`.
25    pub kind: String,
26    pub id: String,
27    pub path: String,
28    pub owner_reason: String,
29    pub status_from: String,
30    pub status_to: String,
31    pub aliases: Vec<String>,
32    pub unresolved: Vec<String>,
33}
34
35#[derive(Debug, Default, Serialize)]
36pub struct ImportPlan {
37    pub rows: Vec<Row>,
38    #[serde(skip)]
39    pub plan: Plan,
40    pub alias_rows: Vec<(String, String)>,
41    /// Attachments over the repository limit: (bundle path, sha256) to upload to the store.
42    pub uploads: Vec<(String, String)>,
43}
44
45fn name_of(p: &str) -> &str {
46    p.rsplit('/').next().unwrap_or(p)
47}
48
49pub fn build(c: &Corpus, map: &ImportMap) -> Result<ImportPlan> {
50    let units = sources::expand(&c.cfg.root, map);
51    let mut out = ImportPlan::default();
52    let mut taken: BTreeSet<String> = c.docs.iter().map(|d| d.id.to_string()).collect();
53    let mut by_source: BTreeMap<String, String> = BTreeMap::new();
54    let mut redirects = vec![];
55    for u in &units {
56        let rule = &map.sources[u.rule];
57        if let Some(re) = &rule.redirect {
58            redirects.push((u.clone(), Regex::new(re).map_err(|e| anyhow!("bad redirect regex: {e}"))?));
59            continue;
60        }
61        let row = doc_row(c, map, u, &mut taken, &mut out)?;
62        by_source.insert(u.main.clone(), row.id.clone());
63        by_source.insert(u.source.clone(), row.id.clone());
64        out.rows.push(row);
65    }
66    for (u, re) in redirects {
67        let text = std::fs::read_to_string(c.cfg.root.join(&u.main)).unwrap_or_default();
68        let target = re
69            .captures(&text)
70            .and_then(|cap| cap.name("target"))
71            .map(|m| m.as_str().trim().trim_start_matches("./").to_string());
72        let resolved = target.as_ref().and_then(|t| {
73            by_source.get(t).cloned().or_else(|| c.docs.iter().find(|d| &d.path == t).map(|d| d.id.to_string()))
74        });
75        let mut unresolved = vec![];
76        match &resolved {
77            Some(id) => out.alias_rows.push((u.source.clone(), id.clone())),
78            None => unresolved.push(format!("stub target {} is not imported", target.unwrap_or_else(|| "?".into()))),
79        }
80        out.plan.remove(u.source.clone());
81        out.rows.push(Row {
82            source: u.source.clone(),
83            kind: "redirect".into(),
84            id: resolved.unwrap_or_default(),
85            path: String::new(),
86            owner_reason: String::new(),
87            status_from: String::new(),
88            status_to: String::new(),
89            aliases: vec![],
90            unresolved,
91        });
92    }
93    Ok(out)
94}
95
96fn doc_row(c: &Corpus, map: &ImportMap, u: &Unit, taken: &mut BTreeSet<String>, out: &mut ImportPlan) -> Result<Row> {
97    let rule = &map.sources[u.rule];
98    let ty_name = rule.ty.clone().ok_or_else(|| anyhow!("source {} needs a type", rule.glob))?;
99    let ty = c.cfg.registry.get(&ty_name).ok_or_else(|| anyhow!("unknown type {ty_name}"))?;
100    let text = std::fs::read_to_string(c.cfg.root.join(&u.main))?;
101    let split = frontmatter::split(&text);
102    let mut unresolved = vec![];
103    let old = match frontmatter::parse(split.raw) {
104        frontmatter::Parsed::Map(m) => m,
105        frontmatter::Parsed::None => Map::new(),
106        frontmatter::Parsed::Invalid(e) => {
107            unresolved.push(format!("old frontmatter does not parse: {e}"));
108            Map::new()
109        }
110    };
111    let scanned = body::scan(split.body, split.body_line, &body::path_cite_re(c.cfg.spec_dir()));
112    let name = name_of(&u.source).trim_end_matches(".md");
113    let name_re = rule.name.as_deref().map(Regex::new).transpose().map_err(|e| anyhow!("bad name regex: {e}"))?;
114    let caps = name_re.as_ref().and_then(|r| r.captures(name_of(&u.source)));
115    let fallback_title = scanned.first_heading.clone().unwrap_or_else(|| name.replace(['-', '_'], " "));
116    let mut conv = convert::frontmatter(map, rule, &old, &ty.statuses, &fallback_title);
117    unresolved.append(&mut conv.unresolved);
118    let title = conv.fm.get("title").and_then(Value::as_str).unwrap_or(&fallback_title).to_string();
119    let created = conv.fm.get("created").and_then(Value::as_str).map(str::to_string);
120    let src =
121        key::Source { path: &u.main, caps: caps.as_ref(), created: created.as_deref(), title: &title, name_stem: name };
122    let (mut k, slug) = key::choose(&c.cfg.root, rule, ty, &src);
123    let governs: Vec<String> = conv
124        .fm
125        .get("rel")
126        .and_then(|r| r.get("governs"))
127        .map(|v| crate::model::doc::list_of(Some(v)))
128        .unwrap_or_default();
129    let (owner, owner_reason) = match rule.owner.as_str() {
130        "auto" => owner::choose(&c.owners, &governs, &text),
131        fixed => {
132            let o = c.owners.by_id(fixed).ok_or_else(|| anyhow!("source {}: no owner '{fixed}'", rule.glob))?;
133            (o, format!("fixed to {fixed} by the import map"))
134        }
135    };
136    loop {
137        let id = DocId::new(&ty.name, &owner.id, &k).to_string();
138        if !taken.contains(&id) {
139            break;
140        }
141        k = match ty.key {
142            KeyKind::Stamp => stamp::next_stamp(&k).unwrap_or(k),
143            KeyKind::Slug => format!("{k}-2"),
144        };
145    }
146    let slug = if ty.key == KeyKind::Slug { k.clone() } else { slug };
147    let id = DocId::new(&ty.name, &owner.id, &k);
148    taken.insert(id.to_string());
149    let bundle = !u.chapters.is_empty() || !u.attachments.is_empty() || ty.form == Form::Dir;
150    let dir = paths::type_dir(&c.cfg, &c.owners, owner, Tier::Committed, ty);
151    let main = paths::main_file(&dir, ty, &paths::stem(ty, &k, &slug), bundle);
152    let bundle_dir = main.rsplit_once('/').map(|x| x.0.to_string()).unwrap_or_default();
153    let base = if u.source == u.main { u.main.trim_end_matches(".md").to_string() } else { u.source.clone() };
154    let mut files = vec![];
155    for ch in &u.chapters {
156        let rel = ch.strip_prefix(&format!("{base}/")).unwrap_or(name_of(ch));
157        out.plan.copy(ch.clone(), format!("{bundle_dir}/{}", rel.replace('/', "-")));
158    }
159    for a in &u.attachments {
160        let rel = a.strip_prefix(&format!("{base}/")).unwrap_or(name_of(a)).to_string();
161        let (sha, bytes) = hash::file(&c.cfg.root.join(a))?;
162        let store = if bytes > c.cfg.raw.files.inline_max_bytes { "store" } else { "repo" };
163        if store == "store" {
164            out.uploads.push((format!("{bundle_dir}/{rel}"), sha.clone()));
165        }
166        out.plan.copy(a.clone(), format!("{bundle_dir}/{rel}"));
167        files.push(json!({"path": rel, "sha256": sha, "bytes": bytes, "store": store}));
168    }
169    let mut fm = conv.fm;
170    fm.insert("id".into(), Value::String(id.to_string()));
171    fm.entry("created").or_insert_with(|| Value::String(stamp::normalize_date(&k).unwrap_or_else(stamp::today)));
172    let created_v = fm["created"].clone();
173    fm.entry("updated").or_insert(created_v);
174    let mut aliases = conv.aliases;
175    if let Some(a) = caps.as_ref().and_then(|c| c.name("alias")) {
176        aliases.push(a.as_str().to_string());
177    }
178    if !aliases.is_empty() {
179        fm.insert("aliases".into(), json!(aliases));
180    }
181    if !files.is_empty() {
182        fm.insert("files".into(), Value::Array(files));
183    }
184    let content = format!("{}{}", frontmatter::render(&crate::markdown::frontmatter::edit::ordered(fm)), split.body);
185    out.plan.write(main.clone(), content, true);
186    out.plan.remove(u.source.clone());
187    for old in std::iter::once(u.source.clone())
188        .chain((u.main != u.source).then(|| u.main.clone()))
189        .chain(aliases.iter().cloned())
190    {
191        out.alias_rows.push((old, id.to_string()));
192    }
193    Ok(Row {
194        source: u.source.clone(),
195        kind: "doc".into(),
196        id: id.to_string(),
197        path: main,
198        owner_reason,
199        status_from: conv.status_from,
200        status_to: fm_status(&out.plan, &id.to_string()),
201        aliases,
202        unresolved,
203    })
204}
205
206fn fm_status(plan: &Plan, _id: &str) -> String {
207    plan.changes
208        .iter()
209        .rev()
210        .find_map(|c| match c {
211            crate::authoring::Change::Write { content, .. } => {
212                content.lines().find_map(|l| l.strip_prefix("status: ").map(str::to_string))
213            }
214            _ => None,
215        })
216        .unwrap_or_default()
217}
218
219/// Appends the alias rows to the table file (creating it), keeping one row per old name.
220pub fn alias_table(existing: &str, rows: &[(String, String)]) -> String {
221    let mut seen: BTreeSet<String> = existing.lines().map(str::to_string).collect();
222    let mut out = if existing.is_empty() {
223        "# old\tnew-id  (written by spec import; frozen afterwards)\n".to_string()
224    } else {
225        existing.to_string()
226    };
227    for (old, new) in rows {
228        let line = format!("{old}\t{new}");
229        if seen.insert(line.clone()) {
230            out.push_str(&line);
231            out.push('\n');
232        }
233    }
234    out
235}
specfold_core/import/plan.rs

specfold_core/import/
plan.rs
