1use super::convert;
4use super::key;
5use super::map::ImportMap;
6use super::owner;
7use super::sources::{self, Unit};
8use crate::authoring::Plan;
9use crate::corpus::Corpus;
10use crate::files::hash;
11use crate::markdown::{body, frontmatter};
12use crate::model::doctype::{Form, KeyKind};
13use crate::model::{DocId, Tier, stamp};
14use crate::placement::paths;
15use anyhow::{Result, anyhow};
16use regex::Regex;
17use serde::Serialize;
18use serde_json::{Map, Value, json};
19use std::collections::{BTreeMap, BTreeSet};
20
21#[derive(Debug, Clone, Default, Serialize)]
22pub struct Row {
23 pub source: String,
24 pub kind: String,
26 pub id: String,
27 pub path: String,
28 pub owner_reason: String,
29 pub status_from: String,
30 pub status_to: String,
31 pub aliases: Vec<String>,
32 pub unresolved: Vec<String>,
33 #[serde(skip_serializing_if = "Vec::is_empty")]
34 pub notes: Vec<String>,
35 #[serde(skip)]
37 pub replaced_by: Option<String>,
38 #[serde(skip)]
40 pub old_main: String,
41}
42
43#[derive(Debug, Default, Serialize)]
44pub struct ImportPlan {
45 pub rows: Vec<Row>,
46 #[serde(skip)]
47 pub plan: Plan,
48 pub alias_rows: Vec<(String, String)>,
49 pub uploads: Vec<(String, String)>,
51}
52
53fn name_of(p: &str) -> &str {
54 p.rsplit('/').next().unwrap_or(p)
55}
56
57fn stem_of(p: &str) -> &str {
59 name_of(p).trim_end_matches(".md")
60}
61
62pub fn build(c: &Corpus, map: &ImportMap) -> Result<ImportPlan> {
63 let units = sources::expand(&c.cfg.root, map);
64 let mut out = ImportPlan::default();
65 let mut taken: BTreeSet<String> = c.docs.iter().map(|d| d.id.to_string()).collect();
66 let mut by_source: BTreeMap<String, String> = BTreeMap::new();
67 let mut redirects = vec![];
68 for u in &units {
69 let rule = &map.sources[u.rule];
70 if let Some(re) = &rule.redirect {
71 redirects.push((u.clone(), Regex::new(re).map_err(|e| anyhow!("bad redirect regex: {e}"))?));
72 continue;
73 }
74 if rule.drop {
75 out.plan.remove(u.source.clone());
76 out.rows.push(Row { source: u.source.clone(), kind: "drop".into(), ..Row::default() });
77 continue;
78 }
79 let row = doc_row(c, map, u, &mut taken, &mut out)?;
80 for p in [&u.main, &u.source].into_iter().chain(&u.chapters).chain(&u.attachments) {
81 by_source.insert(p.clone(), row.id.clone());
82 }
83 if map.stem_aliases {
84 out.alias_rows.push((stem_of(&u.source).to_string(), row.id.clone()));
85 }
86 out.rows.push(row);
87 }
88 for (u, re) in redirects {
89 let text = std::fs::read_to_string(c.cfg.root.join(&u.main)).unwrap_or_default();
90 let target = re
91 .captures(&text)
92 .and_then(|cap| cap.name("target"))
93 .map(|m| m.as_str().trim().trim_start_matches("./").to_string());
94 let resolved = target.as_ref().and_then(|t| {
95 by_source.get(t).cloned().or_else(|| c.docs.iter().find(|d| &d.path == t).map(|d| d.id.to_string()))
96 });
97 let mut unresolved = vec![];
98 match &resolved {
99 Some(id) => {
100 out.alias_rows.push((u.source.clone(), id.clone()));
101 if map.stem_aliases {
102 out.alias_rows.push((stem_of(&u.source).to_string(), id.clone()));
103 }
104 }
105 None => unresolved.push(format!("stub target {} is not imported", target.unwrap_or_else(|| "?".into()))),
106 }
107 out.plan.remove(u.source.clone());
108 out.rows.push(Row {
109 source: u.source.clone(),
110 kind: "redirect".into(),
111 id: resolved.unwrap_or_default(),
112 path: String::new(),
113 owner_reason: String::new(),
114 status_from: String::new(),
115 status_to: String::new(),
116 aliases: vec![],
117 unresolved,
118 ..Row::default()
119 });
120 }
121 canonical_links(&mut out, &by_source);
122 link_replacements(&mut out);
123 rewrite_links(&mut out, &by_source);
124 Ok(out)
125}
126
127fn by_candidates(by: &str) -> Vec<String> {
130 static LINK: std::sync::LazyLock<Regex> =
131 std::sync::LazyLock::new(|| Regex::new(r"\[([^\]]*)\]\(([^)\s]+)\)").unwrap());
132 let mut out = vec![];
133 let mut texts = vec![];
134 for c in LINK.captures_iter(by) {
135 let t = c[2].trim_start_matches("./").to_string();
136 out.push(stem_of(&t).to_string());
137 out.push(t);
138 texts.push(c[1].to_string());
139 }
140 let plain = LINK.replace_all(by, "").trim().to_string();
141 texts.push(plain);
142 for t in texts {
143 let t = t.trim().trim_end_matches(['.', ',']).to_string();
144 if t.is_empty() {
145 continue;
146 }
147 let digits: String =
148 t.chars().rev().take_while(char::is_ascii_digit).collect::<Vec<_>>().into_iter().rev().collect();
149 out.push(t);
150 if !digits.is_empty() {
151 out.push(digits);
152 }
153 }
154 out
155}
156
157fn link_replacements(out: &mut ImportPlan) {
160 let names: BTreeMap<String, String> = out.alias_rows.iter().map(|(o, n)| (o.clone(), n.clone())).collect();
161 let by_name = |c: &str| {
163 let numbered = format!("{c}-");
164 names.get(c).cloned().or_else(|| {
165 names
166 .iter()
167 .find(|(o, _)| {
168 stem_of(o) == c || (c.chars().all(|ch| ch.is_ascii_digit()) && stem_of(o).starts_with(&numbered))
169 })
170 .map(|x| x.1.clone())
171 })
172 };
173 let paths: BTreeMap<String, String> =
174 out.rows.iter().filter(|r| r.kind == "doc").map(|r| (r.id.clone(), r.path.clone())).collect();
175 for i in 0..out.rows.len() {
176 let Some(by) = out.rows[i].replaced_by.clone() else { continue };
177 let old = out.rows[i].id.clone();
178 let newer = by_candidates(&by).iter().find_map(|c| by_name(c)).filter(|n| *n != old && paths.contains_key(n));
179 match newer {
180 Some(n) => {
181 edit_frontmatter(&mut out.plan, &paths[&n], |fm| {
182 let rel = fm.entry("rel").or_insert_with(|| Value::Object(Map::new()));
183 if let Value::Object(rel) = rel
184 && let Value::Array(items) = rel.entry("supersedes").or_insert_with(|| Value::Array(vec![]))
185 && !items.iter().any(|v| v.as_str() == Some(&old))
186 {
187 items.push(Value::String(old.clone()));
188 }
189 });
190 out.rows[i].notes.push(format!("replaced by {n}: rel.supersedes added there"));
191 }
192 None => out.rows[i].unresolved.push(format!(
193 "replaced by {by}, which this import does not include: add rel.supersedes: [{old}] to the newer record"
194 )),
195 }
196 }
197}
198
199fn canonical_links(out: &mut ImportPlan, by_source: &BTreeMap<String, String>) {
203 let mut names: BTreeMap<String, String> = BTreeMap::new();
204 for (src, id) in
205 by_source.iter().map(|(a, b)| (a.as_str(), b)).chain(out.alias_rows.iter().map(|(a, b)| (a.as_str(), b)))
206 {
207 names.entry(stem_of(src).to_string()).or_insert_with(|| id.clone());
208 }
209 let lookup = |v: &str| -> Option<String> {
210 let v = v.trim();
211 if v.contains(':') || v.contains('/') {
212 return None;
213 }
214 names.get(stem_of(v)).cloned()
215 };
216 let paths: Vec<String> = out.rows.iter().filter(|r| r.kind == "doc").map(|r| r.path.clone()).collect();
217 for path in paths {
218 edit_frontmatter(&mut out.plan, &path, |fm| {
219 let Some(Value::Object(rel)) = fm.get_mut("rel") else { return };
220 for (key, v) in rel.iter_mut() {
221 if !crate::graph::DOC_RELATIONS.contains(&key.as_str()) {
222 continue;
223 }
224 match v {
225 Value::String(t) => {
226 if let Some(id) = lookup(t) {
227 *t = id;
228 }
229 }
230 Value::Array(items) => {
231 for item in items.iter_mut() {
232 if let Value::String(t) = item
233 && let Some(id) = lookup(t)
234 {
235 *t = id;
236 }
237 }
238 }
239 _ => {}
240 }
241 }
242 });
243 }
244}
245
246fn edit_frontmatter(plan: &mut Plan, path: &str, f: impl FnOnce(&mut Map<String, Value>)) {
248 for c in plan.changes.iter_mut() {
249 if let crate::authoring::Change::Write { path: p, content, .. } = c
250 && p == path
251 {
252 let split = frontmatter::split(content);
253 let frontmatter::Parsed::Map(mut fm) = frontmatter::parse(split.raw) else { return };
254 f(&mut fm);
255 let body = split.body.to_string();
256 *content = format!("{}{}", frontmatter::render(&crate::markdown::frontmatter::edit::ordered(fm)), body);
257 return;
258 }
259 }
260}
261
262fn relative(from_dir: &str, to: &str) -> String {
264 let f: Vec<&str> = from_dir.split('/').filter(|s| !s.is_empty()).collect();
265 let t: Vec<&str> = to.split('/').filter(|s| !s.is_empty()).collect();
266 let common = f.iter().zip(&t).take_while(|(a, b)| a == b).count();
267 let mut parts: Vec<&str> = std::iter::repeat_n("..", f.len() - common).collect();
268 parts.extend(&t[common..]);
269 parts.join("/")
270}
271
272fn join(dir: &str, link: &str) -> Option<String> {
274 let mut parts: Vec<&str> = dir.split('/').filter(|s| !s.is_empty()).collect();
275 for seg in link.split('/') {
276 match seg {
277 "" | "." => {}
278 ".." => {
279 parts.pop()?;
280 }
281 s => parts.push(s),
282 }
283 }
284 Some(parts.join("/"))
285}
286
287fn dir_of(p: &str) -> &str {
288 p.rsplit_once('/').map(|x| x.0).unwrap_or("")
289}
290
291fn rewrite_links(out: &mut ImportPlan, by_source: &BTreeMap<String, String>) {
294 static LINK: std::sync::LazyLock<Regex> =
295 std::sync::LazyLock::new(|| Regex::new(r"\]\(([^)\s#]+)(#[^)\s]*)?\)").unwrap());
296 let paths: BTreeMap<String, String> =
297 out.rows.iter().filter(|r| r.kind == "doc").map(|r| (r.id.clone(), r.path.clone())).collect();
298 for row in out.rows.iter_mut().filter(|r| r.kind == "doc" && !r.old_main.is_empty()) {
299 let old_dir = dir_of(&row.old_main).to_string();
300 let new_dir = dir_of(&row.path).to_string();
301 let mut count = 0;
302 for c in out.plan.changes.iter_mut() {
303 let crate::authoring::Change::Write { path, content, .. } = c else { continue };
304 if *path != row.path {
305 continue;
306 }
307 let mut fenced = false;
308 let mut lines = vec![];
309 for l in content.split_inclusive('\n') {
310 let t = l.trim_start();
311 if t.starts_with("```") || t.starts_with("~~~") {
312 fenced = !fenced;
313 }
314 if fenced {
315 lines.push(l.to_string());
316 continue;
317 }
318 lines.push(
319 LINK.replace_all(l, |c: ®ex::Captures| {
320 let target = &c[1];
321 let new = (!target.contains("://") && !target.starts_with('/'))
322 .then(|| join(&old_dir, target))
323 .flatten()
324 .and_then(|old| by_source.get(&old))
325 .and_then(|id| paths.get(id));
326 match new {
327 Some(p) => {
328 count += 1;
329 format!("]({}{})", relative(&new_dir, p), c.get(2).map_or("", |m| m.as_str()))
330 }
331 None => c[0].to_string(),
332 }
333 })
334 .into_owned(),
335 );
336 }
337 *content = lines.concat();
338 }
339 if count > 0 {
340 row.notes.push(format!("{count} link(s) to old file names now point at the new paths"));
341 }
342 }
343}
344
345fn without_section(body: &str, name: &str) -> Option<String> {
347 let is_it = |l: &str| l.starts_with('#') && l.trim_start_matches('#').trim().eq_ignore_ascii_case(name);
348 let lines: Vec<&str> = body.split_inclusive('\n').collect();
349 let at = lines.iter().position(|l| is_it(l.trim()))?;
350 let end = lines[at + 1..].iter().position(|l| l.starts_with('#')).map_or(lines.len(), |e| at + 1 + e);
351 Some([&lines[..at], &lines[end..]].concat().concat())
352}
353
354fn doc_row(c: &Corpus, map: &ImportMap, u: &Unit, taken: &mut BTreeSet<String>, out: &mut ImportPlan) -> Result<Row> {
355 let rule = &map.sources[u.rule];
356 let ty_name = rule.ty.clone().ok_or_else(|| anyhow!("source {} needs a type", rule.glob))?;
357 let ty = c.cfg.registry.get(&ty_name).ok_or_else(|| anyhow!("unknown type {ty_name}"))?;
358 let generated = u.main.is_empty();
360 let text = if generated { String::new() } else { std::fs::read_to_string(c.cfg.root.join(&u.main))? };
361 let split = frontmatter::split(&text);
362 let mut unresolved = vec![];
363 let old = match frontmatter::parse(split.raw) {
364 frontmatter::Parsed::Map(m) => m,
365 frontmatter::Parsed::None => Map::new(),
366 frontmatter::Parsed::Invalid(e) => {
367 unresolved.push(format!("old frontmatter does not parse: {e}"));
368 Map::new()
369 }
370 };
371 let mut old = old;
372 let mut body_links = vec![];
373 if !generated {
374 let found = super::body_meta::scan(split.body, &map.body);
375 let has_status = old.contains_key("status");
377 let has_date = old.contains_key("date") || old.contains_key("created");
378 if let Some(st) = found.status
379 && !old.contains_key("status")
380 {
381 old.insert("status".into(), Value::String(st));
382 }
383 if let Some(d) = found.date
384 && !old.contains_key("date")
385 && !old.contains_key("created")
386 {
387 old.insert("date".into(), Value::String(d));
388 }
389 body_links = found.links;
390 unresolved.extend(found.missed.into_iter().filter(|m| {
391 !(has_status && m.starts_with("no status in the body") || has_date && m.starts_with("no date in the body"))
392 }));
393 }
394 let scanned = body::scan(split.body, split.body_line, &body::path_cite_re(c.cfg.spec_dir()));
395 let name = name_of(&u.source).trim_end_matches(".md");
396 let name_re = rule.name.as_deref().map(Regex::new).transpose().map_err(|e| anyhow!("bad name regex: {e}"))?;
397 let caps = name_re.as_ref().and_then(|r| r.captures(name_of(&u.source)));
398 let named = caps.as_ref().and_then(|c| c.name("slug")).map(|m| m.as_str()).unwrap_or(name);
399 let heading = scanned.first_heading.clone().map(|h| if map.strip_title_number { strip_number(&h) } else { h });
400 let fallback_title = heading.unwrap_or_else(|| named.replace(['-', '_'], " "));
401 let mut conv = convert::frontmatter(map, rule, &old, &ty.statuses, &fallback_title);
402 unresolved.append(&mut conv.unresolved);
403 if !body_links.is_empty() {
404 let rel = conv.fm.entry("rel").or_insert_with(|| Value::Object(Map::new()));
405 if let Value::Object(rel) = rel {
406 for (kind, target) in body_links {
407 let name = target.rsplit('/').next().unwrap_or(&target).trim_end_matches(".md").to_string();
408 if let Value::Array(items) = rel.entry(kind).or_insert_with(|| Value::Array(vec![]))
409 && !items.iter().any(|v| v.as_str() == Some(&name))
410 {
411 items.push(Value::String(name));
412 }
413 }
414 }
415 }
416 let title = conv.fm.get("title").and_then(Value::as_str).unwrap_or(&fallback_title).to_string();
417 let created = conv.fm.get("created").and_then(Value::as_str).map(str::to_string);
418 let src =
419 key::Source { path: &u.main, caps: caps.as_ref(), created: created.as_deref(), title: &title, name_stem: name };
420 let (mut k, slug) = key::choose(&c.cfg.root, rule, ty, &src);
421 let governs: Vec<String> = conv
422 .fm
423 .get("rel")
424 .and_then(|r| r.get("governs"))
425 .map(|v| crate::model::doc::list_of(Some(v)))
426 .unwrap_or_default();
427 let (owner, owner_reason) = match rule.owner.as_str() {
428 "auto" => {
429 let home = c.owners.storage_owner(c.owners.containing(&u.source));
430 owner::choose(&c.owners, &governs, &text, home)
431 }
432 fixed => {
433 let o = c.owners.by_id(fixed).ok_or_else(|| anyhow!("source {}: no owner '{fixed}'", rule.glob))?;
434 (o, format!("fixed to {fixed} by the import map"))
435 }
436 };
437 loop {
438 let id = DocId::new(&ty.name, &owner.id, &k).to_string();
439 if !taken.contains(&id) {
440 break;
441 }
442 k = match ty.key {
443 KeyKind::Stamp => stamp::next_stamp(&k).unwrap_or(k),
444 KeyKind::Slug => format!("{k}-2"),
445 };
446 }
447 let slug = if ty.key == KeyKind::Slug { k.clone() } else { slug };
448 let id = DocId::new(&ty.name, &owner.id, &k);
449 taken.insert(id.to_string());
450 let bundle = !u.chapters.is_empty() || !u.attachments.is_empty() || ty.form == Form::Dir;
451 let dir = paths::type_dir(&c.cfg, &c.owners, owner, Tier::Committed, ty);
452 let main = paths::main_file(&dir, ty, &paths::stem(ty, &k, &slug), bundle);
453 let bundle_dir = main.rsplit_once('/').map(|x| x.0.to_string()).unwrap_or_default();
454 let base = if u.source == u.main { u.main.trim_end_matches(".md").to_string() } else { u.source.clone() };
455 let mut files = vec![];
456 for ch in &u.chapters {
459 let rel = ch.strip_prefix(&format!("{base}/")).unwrap_or(name_of(ch));
460 out.plan.copy(ch.clone(), format!("{bundle_dir}/{}", rel.replace('/', "-")));
461 out.plan.remove(ch.clone());
462 }
463 for a in &u.attachments {
464 let rel = a.strip_prefix(&format!("{base}/")).unwrap_or(name_of(a)).to_string();
465 let (sha, bytes) = hash::file(&c.cfg.root.join(a))?;
466 let store = if bytes > c.cfg.raw.files.inline_max_bytes { "store" } else { "repo" };
467 if store == "store" {
468 out.uploads.push((format!("{bundle_dir}/{rel}"), sha.clone()));
469 }
470 out.plan.copy(a.clone(), format!("{bundle_dir}/{rel}"));
471 out.plan.remove(a.clone());
472 files.push(json!({"path": rel, "sha256": sha, "bytes": bytes, "store": store}));
473 }
474 let mut fm = conv.fm;
475 fm.insert("id".into(), Value::String(id.to_string()));
476 fm.entry("created").or_insert_with(|| Value::String(stamp::normalize_date(&k).unwrap_or_else(stamp::today)));
477 let created_v = fm["created"].clone();
478 let updated = fm.entry("updated").or_insert(created_v.clone());
479 if updated.as_str() < created_v.as_str() {
481 *updated = created_v;
482 }
483 let mut aliases = conv.aliases;
484 if let Some(a) = caps.as_ref().and_then(|c| c.name("alias")) {
485 aliases.push(a.as_str().to_string());
486 }
487 if !aliases.is_empty() {
488 fm.insert("aliases".into(), json!(aliases));
489 }
490 if !files.is_empty() {
491 fm.insert("files".into(), Value::Array(files));
492 }
493 let mut notes = conv.notes;
494 let mut body = if generated { format!("\n# {title}\n") } else { split.body.to_string() };
495 if let Some(name) = map.body.status.as_deref().and_then(|h| h.strip_prefix("section:"))
496 && let Some(rest) = without_section(&body, name)
497 {
498 if map.body.drop_status_section {
499 body = rest;
500 } else {
501 notes.push(format!(
502 "the body keeps '## {name}' next to frontmatter status; [body] drop_status_section = true removes it"
503 ));
504 }
505 }
506 let alnum = named.chars().filter(|c| c.is_alphanumeric() && !c.is_ascii_digit()).count();
508 let ascii = named.chars().filter(|c| c.is_ascii_alphabetic()).count();
509 if alnum > 0 && ascii * 2 < alnum {
510 notes.push(format!(
511 "the name is mostly non-ASCII, so the slug is '{slug}' (the title stays); to rename: spec mv {id} --slug <slug> --write"
512 ));
513 }
514 let content = format!("{}{}", frontmatter::render(&crate::markdown::frontmatter::edit::ordered(fm)), body);
515 crate::authoring::forbid::refuse(c, &main, crate::authoring::forbid::NO_LOCAL_HINT)?;
516 out.plan.write(main.clone(), content, true);
517 out.plan.remove(u.source.clone());
518 for old in std::iter::once(u.source.clone())
520 .chain((u.main != u.source).then(|| u.main.clone()))
521 .chain(u.chapters.iter().cloned())
522 .chain(u.attachments.iter().cloned())
523 .chain(aliases.iter().cloned())
524 {
525 out.alias_rows.push((old, id.to_string()));
526 }
527 Ok(Row {
528 source: u.source.clone(),
529 kind: "doc".into(),
530 id: id.to_string(),
531 path: main,
532 owner_reason,
533 status_from: conv.status_from,
534 status_to: fm_status(&out.plan, &id.to_string()),
535 aliases,
536 unresolved,
537 notes,
538 replaced_by: conv.replaced_by,
539 old_main: if generated { String::new() } else { u.main.clone() },
540 })
541}
542
543fn strip_number(title: &str) -> String {
545 let digits = title.chars().take_while(char::is_ascii_digit).count();
546 match title[digits..].strip_prefix('.') {
547 Some(rest) if digits > 0 && !rest.trim().is_empty() => rest.trim().to_string(),
548 _ => title.to_string(),
549 }
550}
551
552fn fm_status(plan: &Plan, _id: &str) -> String {
553 plan.changes
554 .iter()
555 .rev()
556 .find_map(|c| match c {
557 crate::authoring::Change::Write { content, .. } => {
558 content.lines().find_map(|l| l.strip_prefix("status: ").map(str::to_string))
559 }
560 _ => None,
561 })
562 .unwrap_or_default()
563}
564
565pub fn alias_table(existing: &str, rows: &[(String, String)]) -> String {
567 let mut seen: BTreeSet<String> = existing.lines().map(str::to_string).collect();
568 let mut out = if existing.is_empty() {
569 "# old\tnew-id (written by spec import; frozen afterwards)\n".to_string()
570 } else {
571 existing.to_string()
572 };
573 for (old, new) in rows {
574 let line = format!("{old}\t{new}");
575 if seen.insert(line.clone()) {
576 out.push_str(&line);
577 out.push('\n');
578 }
579 }
580 out
581}
582
583#[cfg(test)]
584mod tests {
585 #[test]
586 fn record_numbers_come_off_titles() {
587 assert_eq!(super::strip_number("2. Use X"), "Use X");
588 assert_eq!(super::strip_number("12. "), "12. ");
589 assert_eq!(super::strip_number("2026 plan"), "2026 plan");
590 }
591
592 #[test]
593 fn sections_come_out_and_links_are_made_relative() {
594 let body = "\n# T\n\n## Status\n\nAccepted\n\n## Context\n\nx\n";
595 assert_eq!(super::without_section(body, "status").unwrap(), "\n# T\n\n## Context\n\nx\n");
596 assert_eq!(super::relative(".spec/adr", ".spec/adr/a.adr.md"), "a.adr.md");
597 assert_eq!(super::relative("svc/.spec/adr", ".spec/adr/a.md"), "../../../.spec/adr/a.md");
598 assert_eq!(super::join("doc/adr", "../x/./y.md").as_deref(), Some("doc/x/y.md"));
599 assert_eq!(super::join("", "../y.md"), None);
600 assert_eq!(
601 super::by_candidates("[ADR-0002](0002-use-y.md)"),
602 ["0002-use-y", "0002-use-y.md", "ADR-0002", "0002"]
603 );
604 }
605}