Skip to main content

nb_api/transaction/
commit.rs

1use std::collections::HashSet;
2use std::path::{Path, PathBuf};
3
4use crate::error::NbError;
5use crate::gate;
6use crate::git;
7use crate::types::{CommitOutcome, OpOutcome};
8
9use super::apply::{OpMeta, PlanEffects, validate_and_apply_virtual};
10use super::index::{
11    PendingIndexEdit, acquire_index_lock, apply_index_edits, index_id_on_disk,
12    is_index_rel, numeric_selector, reap_stale_index_lock,
13};
14use super::plan::annotate_plan_index;
15use super::virtual_tree::{
16    VirtualNode, VirtualTree, parent_path, refuse_symlink_ancestors,
17};
18
19use super::plan::Transaction;
20
21impl Transaction {
22    /// Validate-all, apply-all, at most one Git checkpoint.
23    pub async fn commit(self) -> Result<CommitOutcome, NbError> {
24        // Resolve root without invoking `nb` when possible (avoids nb
25        // auto-checkpoint clearing dirty state before the baseline check).
26        let notebook_root = self
27            .client
28            .show_notebook_path_unguarded(Some(&self.notebook))
29            .await?;
30        let gate_key = gate::git_common_dir_realpath(&notebook_root)?;
31        let _hold =
32            gate::acquire_notebook(gate_key, self.gate_timeout, false).await?;
33
34        // A crashed writer may leave its `.nb-api-index.lock` behind; an
35        // untracked lock file reads as dirty, so reap a stale one first.
36        reap_stale_index_lock(&notebook_root, self.gate_timeout);
37
38        if git::notebook_is_dirty(&notebook_root)? {
39            // Index-only reconciliation droppings (e.g. an external
40            // `nb` background checkpoint racing our gate) are
41            // absorbed by our own checkpoint below; anything else
42            // still refuses. See `git::status_is_index_only`.
43            let status = git::notebook_status_porcelain(&notebook_root)?;
44            if !git::status_is_index_only(&status) {
45                return Err(NbError::DirtyBaseline {
46                    guidance: "commit or clean the notebook worktree/index before Transaction::commit"
47                        .into(),
48                });
49            }
50        }
51
52        let pre_revision = git::notebook_head(&notebook_root)?;
53        if self.plan.is_empty() {
54            return Ok(CommitOutcome {
55                commit_created: false,
56                revision_id: None,
57                pre_revision,
58                ops: Vec::new(),
59            });
60        }
61
62        let ignored_existing: HashSet<String> =
63            git::list_ignored_paths(&notebook_root)?
64                .into_iter()
65                .collect();
66        let mut tree = VirtualTree::from_disk(&notebook_root)?;
67        let mut op_meta: Vec<OpMeta> = Vec::with_capacity(self.plan.len());
68        let mut effects = PlanEffects::default();
69        // Disk keys at snapshot time: materialize refuses a planned create
70        // that appears on disk afterwards (external same-path race).
71        let snapshot_keys: HashSet<String> =
72            tree.nodes.keys().cloned().collect();
73
74        for (index, op) in self.plan.iter().enumerate() {
75            match validate_and_apply_virtual(
76                &self.notebook,
77                &notebook_root,
78                &mut tree,
79                &ignored_existing,
80                op,
81                index,
82                &mut effects,
83            ) {
84                Ok(meta) => op_meta.push(meta),
85                Err(err) => {
86                    return Err(annotate_plan_index(err, index as u32));
87                },
88            }
89        }
90
91        #[cfg(feature = "testing")]
92        if std::env::var_os("NB_API_SIMULATE_EXTERNAL_WRITER").is_some() {
93            // Deterministic stand-in for an external `nb add` racing our
94            // snapshot: create AND commit a note this transaction knows
95            // nothing about. Our commit must preserve it (file + index line).
96            simulate_external_writer(&notebook_root)?;
97        }
98
99        // Disk baseline before materialize (for rollback of new owned outputs).
100        // Virtual tree already includes planned ops; disk is still pre-apply.
101        let baseline_on_disk: HashSet<String> = {
102            let mut s: HashSet<String> =
103                git::list_notebook_paths(&notebook_root)?
104                    .into_iter()
105                    .collect();
106            s.extend(ignored_existing.iter().cloned());
107            s
108        };
109
110        // Force-stage every final file path so ignore rules cannot drop
111        // transaction-owned outputs (new ignored names, `.gitkeep`, etc.).
112        // `.index` files are applied post-materialize (fresh read under the
113        // index lock) but staged by the same checkpoint, so list them too.
114        let mut force_paths: Vec<String> = tree
115            .nodes
116            .iter()
117            .filter_map(|(p, n)| match n {
118                VirtualNode::File(_) => Some(p.clone()),
119                VirtualNode::Folder => None,
120            })
121            .collect();
122        for meta in &op_meta {
123            for edit in &meta.index_edits {
124                if !force_paths.iter().any(|p| p == &edit.file_rel) {
125                    force_paths.push(edit.file_rel.clone());
126                }
127            }
128        }
129
130        // Directories that already exist before materialize must never be
131        // pruned during rollback (including empty ignored parents).
132        let baseline_dirs = existing_dirs_under(&notebook_root);
133
134        // Materialize non-index files, apply `.index` edits from fresh
135        // on-disk content under `.nb-api-index.lock` (released before git
136        // staging — never held across `git add`/`commit`), then checkpoint.
137        // The lock serializes nb-api writers only; external `nb` writers are
138        // covered by O_APPEND plus post-write selector derivation.
139        let apply_result = async {
140            materialize_tree(&notebook_root, &tree, &effects, &snapshot_keys)?;
141            let mut pending: Vec<(u32, PendingIndexEdit)> = Vec::new();
142            for (i, meta) in op_meta.iter().enumerate() {
143                for edit in &meta.index_edits {
144                    pending.push((i as u32, edit.clone()));
145                }
146            }
147            let placements = if pending.is_empty() {
148                Vec::new()
149            } else {
150                let _lock =
151                    acquire_index_lock(&notebook_root, self.gate_timeout)
152                        .await?;
153                let placements = apply_index_edits(
154                    &notebook_root,
155                    &pending,
156                    self.gate_timeout,
157                )?;
158                drop(_lock);
159                placements
160            };
161            let created = git::notebook_commit_all(
162                &notebook_root,
163                &format!("nb-api transaction ({} ops)", self.plan.len()),
164                &force_paths,
165            )?;
166            Ok::<_, NbError>((created, placements))
167        }
168        .await;
169
170        match apply_result {
171            Ok((created, placements)) => {
172                let revision_id = if created {
173                    match git::notebook_head(&notebook_root) {
174                        Ok(head) => Some(head),
175                        Err(_) => {
176                            return Err(NbError::IndeterminateCommit {
177                                pre_revision,
178                                post_revision_observed: None,
179                                guidance: "git commit may have succeeded but HEAD could not be re-read; do not retry blindly".into(),
180                            });
181                        },
182                    }
183                } else {
184                    None
185                };
186                let ops = op_meta
187                    .into_iter()
188                    .enumerate()
189                    .map(|(i, meta)| {
190                        // Creates/moves/deletes carry post-write placements:
191                        // selector becomes the numeric `<folder>/<id>` form.
192                        // A cross-folder move records both a blank (source)
193                        // and an append (destination); the destination
194                        // placement is last, so take the last match.
195                        // All other ops keep the echoed target selector and
196                        // gain a numeric id when the path is indexed.
197                        let (selector, numeric_id) = match placements
198                            .iter()
199                            .rev()
200                            .find(|p| p.op_index == i as u32)
201                        {
202                            Some(p) => (
203                                Some(numeric_selector(
204                                    &self.notebook,
205                                    &p.folder,
206                                    p.id,
207                                )),
208                                Some(p.id),
209                            ),
210                            None => (
211                                meta.selector,
212                                meta.path.as_deref().and_then(|p| {
213                                    index_id_on_disk(&notebook_root, p)
214                                }),
215                            ),
216                        };
217                        OpOutcome {
218                            index: i as u32,
219                            path: meta.path,
220                            selector,
221                            numeric_id,
222                            noop: meta.noop,
223                            fingerprint: meta.fingerprint,
224                        }
225                    })
226                    .collect();
227                Ok(CommitOutcome {
228                    commit_created: created,
229                    revision_id,
230                    pre_revision,
231                    ops,
232                })
233            },
234            Err(err) => {
235                let post = git::notebook_head(&notebook_root).ok();
236                if post.as_deref().is_some_and(|h| h != pre_revision.as_str())
237                {
238                    return Err(NbError::IndeterminateCommit {
239                        pre_revision,
240                        post_revision_observed: post,
241                        guidance: "checkpoint may have completed; re-read HEAD/status and do not retry the same plan blindly".into(),
242                    });
243                }
244                // Owned outputs that did not exist at baseline must be removed
245                // even when ignored (git clean -fd keeps ignored files).
246                // Baseline directories are never owned outputs: pruning one
247                // would `remove_dir_all` a pre-existing folder.
248                let owned_new: Vec<String> = force_paths
249                    .into_iter()
250                    .filter(|p| !baseline_on_disk.contains(p))
251                    .filter(|p| !baseline_dirs.contains(p))
252                    .collect();
253                match try_restore(
254                    &notebook_root,
255                    &pre_revision,
256                    &owned_new,
257                    &baseline_dirs,
258                ) {
259                    Ok(()) => Err(err),
260                    Err(recovery) => Err(recovery),
261                }
262            },
263        }
264    }
265}
266
267/// All directory paths under the notebook root before materialize (relative,
268/// `/`-separated). Used so rollback never deletes pre-existing dirs.
269fn existing_dirs_under(notebook_root: &Path) -> HashSet<String> {
270    let mut dirs = HashSet::new();
271    fn walk(base: &Path, rel: &Path, out: &mut HashSet<String>) {
272        let entries = match std::fs::read_dir(base) {
273            Ok(e) => e,
274            Err(_) => return,
275        };
276        for entry in entries.flatten() {
277            let name = entry.file_name();
278            if name == ".git" {
279                continue;
280            }
281            let Ok(ft) = entry.file_type() else {
282                continue;
283            };
284            // Do not follow symlinks.
285            if ft.is_symlink() || !ft.is_dir() {
286                continue;
287            }
288            let child_rel = if rel.as_os_str().is_empty() {
289                PathBuf::from(&name)
290            } else {
291                rel.join(&name)
292            };
293            let key = {
294                let s = child_rel.to_string_lossy();
295                // Platform-scoped: Windows disk separators become
296                // `/`; Unix backslashes stay literal so rollback
297                // restores the original directory name.
298                #[cfg(windows)]
299                {
300                    s.replace('\\', "/")
301                }
302                #[cfg(not(windows))]
303                {
304                    s.into_owned()
305                }
306            };
307            out.insert(key);
308            walk(&entry.path(), &child_rel, out);
309        }
310    }
311    walk(notebook_root, Path::new(""), &mut dirs);
312    dirs
313}
314
315fn try_restore(
316    notebook_root: &Path,
317    pre_revision: &str,
318    remove_new_owned: &[String],
319    baseline_dirs: &HashSet<String>,
320) -> Result<(), NbError> {
321    if let Err(e) = git::notebook_reset_clean(notebook_root, pre_revision) {
322        let post = git::notebook_head(notebook_root).ok();
323        let status = git::notebook_status_porcelain(notebook_root).ok();
324        return Err(NbError::RecoveryRequired {
325            pre_revision: pre_revision.to_string(),
326            post_revision_observed: post,
327            status_observed: status,
328            preserved_paths: Some(remove_new_owned.to_vec()),
329            guidance: format!(
330                "cleanup after failed commit could not be verified; inspect HEAD/status before retry. underlying: {e}"
331            ),
332        });
333    }
334    // `git clean -fd` can remove empty untracked dirs (including ignored
335    // parents) once staged children are cleared by reset --hard. Restore any
336    // baseline directory that disappeared.
337    for d in baseline_dirs {
338        let abs = notebook_root.join(d);
339        if !abs.exists()
340            && let Err(e) = std::fs::create_dir_all(&abs)
341        {
342            return Err(NbError::RecoveryRequired {
343                pre_revision: pre_revision.to_string(),
344                post_revision_observed: git::notebook_head(notebook_root).ok(),
345                status_observed: git::notebook_status_porcelain(notebook_root)
346                    .ok(),
347                preserved_paths: Some(vec![d.clone()]),
348                guidance: format!(
349                    "failed to restore baseline directory `{d}` after cleanup; do not retry blindly. underlying: {e}"
350                ),
351            });
352        }
353    }
354    // Explicitly delete transaction-owned outputs that reset/clean leave behind
355    // when they are Git-ignored untracked files.
356    let mut remaining = Vec::new();
357    for rel in remove_new_owned {
358        let abs = notebook_root.join(rel);
359        match std::fs::symlink_metadata(&abs) {
360            Ok(meta) if meta.file_type().is_symlink() || meta.is_file() => {
361                if let Err(e) = std::fs::remove_file(&abs) {
362                    remaining.push(format!("{rel} ({e})"));
363                }
364            },
365            Ok(meta) if meta.is_dir() => {
366                if let Err(e) = std::fs::remove_dir_all(&abs) {
367                    remaining.push(format!("{rel} ({e})"));
368                }
369            },
370            Ok(_) => {},
371            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {},
372            Err(e) => remaining.push(format!("{rel} ({e})")),
373        }
374        // Prune only empty parents that the transaction created — never a
375        // directory that existed at baseline (e.g. pre-existing ignored parent).
376        if let Some(parent) = parent_path(rel) {
377            let mut cur = parent;
378            loop {
379                if baseline_dirs.contains(&cur) {
380                    break;
381                }
382                let p = notebook_root.join(&cur);
383                match std::fs::remove_dir(&p) {
384                    Ok(()) => {},
385                    Err(_) => break,
386                }
387                match parent_path(&cur) {
388                    Some(next) => cur = next,
389                    None => break,
390                }
391            }
392        }
393    }
394    if !remaining.is_empty() {
395        return Err(NbError::RecoveryRequired {
396            pre_revision: pre_revision.to_string(),
397            post_revision_observed: git::notebook_head(notebook_root).ok(),
398            status_observed: git::notebook_status_porcelain(notebook_root).ok(),
399            preserved_paths: Some(remaining),
400            guidance: "failed to remove transaction-owned outputs after aborted commit; do not retry blindly".into(),
401        });
402    }
403    // Verify none of the new owned paths remain (including ignored).
404    let mut still_present = Vec::new();
405    for rel in remove_new_owned {
406        if notebook_root.join(rel).exists() {
407            still_present.push(rel.clone());
408        }
409    }
410    let head = match restore_verify_head(notebook_root) {
411        Ok(h) => h,
412        Err(e) => {
413            return Err(NbError::RecoveryRequired {
414                pre_revision: pre_revision.to_string(),
415                post_revision_observed: None,
416                status_observed: git::notebook_status_porcelain(notebook_root)
417                    .ok(),
418                preserved_paths: if still_present.is_empty() {
419                    None
420                } else {
421                    Some(still_present)
422                },
423                guidance: format!(
424                    "cleanup ran but HEAD re-read failed during verification; do not retry blindly. underlying: {e}"
425                ),
426            });
427        },
428    };
429    let dirty = match restore_verify_dirty(notebook_root) {
430        Ok(d) => d,
431        Err(e) => {
432            return Err(NbError::RecoveryRequired {
433                pre_revision: pre_revision.to_string(),
434                post_revision_observed: Some(head),
435                status_observed: git::notebook_status_porcelain(notebook_root)
436                    .ok(),
437                preserved_paths: if still_present.is_empty() {
438                    None
439                } else {
440                    Some(still_present)
441                },
442                guidance: format!(
443                    "cleanup ran but dirty-status check failed during verification; do not retry blindly. underlying: {e}"
444                ),
445            });
446        },
447    };
448    if head != pre_revision || dirty || !still_present.is_empty() {
449        return Err(NbError::RecoveryRequired {
450            pre_revision: pre_revision.to_string(),
451            post_revision_observed: Some(head),
452            status_observed: git::notebook_status_porcelain(notebook_root).ok(),
453            preserved_paths: if still_present.is_empty() {
454                None
455            } else {
456                Some(still_present)
457            },
458            guidance:
459                "cleanup ran but HEAD/status/owned-path verification failed; do not retry blindly"
460                    .into(),
461        });
462    }
463    Ok(())
464}
465
466/// HEAD observation used only in post-cleanup verification.
467fn restore_verify_head(notebook_root: &Path) -> Result<String, NbError> {
468    #[cfg(feature = "testing")]
469    if std::env::var_os("NB_API_FAIL_RESTORE_HEAD").is_some() {
470        return Err(NbError::CommandFailed {
471            command: "nb-api://fail-restore-head".into(),
472            stderr: "injected HEAD verify failure for rollback tests".into(),
473            exit_code: Some(1),
474        });
475    }
476    git::notebook_head(notebook_root)
477}
478
479/// Dirty-status observation used only in post-cleanup verification.
480fn restore_verify_dirty(notebook_root: &Path) -> Result<bool, NbError> {
481    #[cfg(feature = "testing")]
482    if std::env::var_os("NB_API_FAIL_RESTORE_DIRTY").is_some() {
483        return Err(NbError::CommandFailed {
484            command: "nb-api://fail-restore-dirty".into(),
485            stderr: "injected dirty-status verify failure for rollback tests"
486                .into(),
487            exit_code: Some(1),
488        });
489    }
490    git::notebook_is_dirty(notebook_root)
491}
492
493/// Deterministic stand-in for an external `nb add` racing our snapshot
494/// (testing only, behind `NB_API_SIMULATE_EXTERNAL_WRITER`): create AND
495/// commit a note plus its `.index` line that this transaction knows nothing
496/// about. The commit must preserve both.
497///
498/// `NB_API_SIMULATE_EXTERNAL_SAME_PATH=<rel>` makes the external writer take
499/// the given path instead (to test the same-path race: our commit must fail
500/// `PathCollision`, never overwrite).
501#[cfg(feature = "testing")]
502fn simulate_external_writer(notebook_root: &Path) -> Result<(), NbError> {
503    use std::fs::OpenOptions;
504    use std::io::Write as _;
505
506    use super::index::{index_rel_for_folder, split_folder_basename};
507    let (folder, name) =
508        match std::env::var("NB_API_SIMULATE_EXTERNAL_SAME_PATH") {
509            Ok(rel) => {
510                let (folder, basename) = split_folder_basename(
511                    &super::virtual_tree::normalize_rel(&rel),
512                );
513                (folder, basename)
514            },
515            Err(_) => {
516                (String::new(), format!("external-{}.md", std::process::id()))
517            },
518        };
519    let rel = if folder.is_empty() {
520        name.clone()
521    } else {
522        format!("{folder}/{name}")
523    };
524    if !folder.is_empty() {
525        std::fs::create_dir_all(notebook_root.join(&folder)).map_err(|e| {
526            NbError::Io {
527                path: notebook_root.join(&folder),
528                source: e.into(),
529            }
530        })?;
531    }
532    std::fs::write(notebook_root.join(&rel), b"# External\n\nexternal body\n")
533        .map_err(|e| NbError::Io {
534            path: notebook_root.join(&rel),
535            source: e.into(),
536        })?;
537    let index_abs = notebook_root.join(index_rel_for_folder(&folder));
538    let mut f = OpenOptions::new()
539        .create(true)
540        .append(true)
541        .open(&index_abs)
542        .map_err(|e| NbError::Io {
543            path: index_abs.clone(),
544            source: e.into(),
545        })?;
546    writeln!(f, "{name}").map_err(|e| NbError::Io {
547        path: index_abs.clone(),
548        source: e.into(),
549    })?;
550    drop(f);
551    git::notebook_commit_all(
552        notebook_root,
553        "external writer",
554        &[rel, index_rel_for_folder(&folder)],
555    )?;
556    Ok(())
557}
558
559/// Write the plan's effects to disk.
560///
561/// ONLY paths recorded in `effects` are touched: files created after the
562/// snapshot by an external writer are never deleted, and files deleted
563/// externally are never resurrected. `.index` rels are skipped here (applied
564/// separately from fresh on-disk content under `.nb-api-index.lock`).
565///
566/// `snapshot_keys` are the tree keys at snapshot time. A `created` path that
567/// exists on disk now but was absent from the snapshot means an external
568/// writer created the same path after our collision validation — refuse with
569/// `PathCollision` instead of overwriting it (which would also leave
570/// ambiguous duplicate `.index` entries).
571fn materialize_tree(
572    root: &Path,
573    tree: &VirtualTree,
574    effects: &PlanEffects,
575    snapshot_keys: &HashSet<String>,
576) -> Result<(), NbError> {
577    // Preflight: no path may traverse an existing symlink ancestor (including
578    // ignored directory symlinks excluded from the virtual snapshot).
579    for rel in tree.nodes.keys() {
580        refuse_symlink_ancestors(root, rel)?;
581    }
582
583    // Remove ONLY plan delete/move sources that exist on disk as files.
584    let mut removed: Vec<&String> = effects.removed.iter().collect();
585    removed.sort();
586    removed.dedup();
587    for rel in removed {
588        // Tree keys are already canonical relative paths; use them
589        // verbatim so a literal backslash name never collapses onto
590        // a nested path.
591        let key: String = rel.clone();
592        let abs = root.join(&key);
593        let meta = match std::fs::symlink_metadata(&abs) {
594            Ok(m) => m,
595            Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
596            Err(e) => {
597                return Err(NbError::Io {
598                    path: abs,
599                    source: e.into(),
600                });
601            },
602        };
603        if meta.file_type().is_symlink() {
604            return Err(NbError::UnsupportedStructure {
605                reason: format!(
606                    "refusing to materialize through symlink `{key}`"
607                ),
608            });
609        }
610        if meta.is_dir() {
611            // Plan folder-delete source: remove the whole subtree.
612            // (Move sources are always files; folder moves are
613            // refused at plan validation.)
614            std::fs::remove_dir_all(&abs).map_err(|e| NbError::Io {
615                path: abs,
616                source: e.into(),
617            })?;
618            continue;
619        }
620        if meta.is_file()
621            && !matches!(tree.nodes.get(&key), Some(VirtualNode::File(_)))
622        {
623            std::fs::remove_file(&abs).map_err(|e| NbError::Io {
624                path: abs,
625                source: e.into(),
626            })?;
627        }
628    }
629    // Create ONLY plan folders.
630    let mut mkdirs: Vec<&String> = effects.mkdirs.iter().collect();
631    mkdirs.sort();
632    mkdirs.dedup();
633    for rel in mkdirs {
634        let key: String = rel.clone();
635        let abs = root.join(&key);
636        refuse_symlink_ancestors(root, &key)?;
637        std::fs::create_dir_all(&abs).map_err(|e| NbError::Io {
638            path: abs.clone(),
639            source: e.into(),
640        })?;
641    }
642    // Write ONLY plan-created/modified files (never `.index`).
643    let mut written: Vec<&String> = effects
644        .written
645        .iter()
646        .filter(|rel| !is_index_rel(rel))
647        .collect();
648    written.sort();
649    written.dedup();
650    let created: HashSet<&String> = effects.created.iter().collect();
651    for rel in written {
652        let key: String = rel.clone();
653        let created_contains = created.contains(rel);
654        let Some(VirtualNode::File(bytes)) = tree.nodes.get(&key) else {
655            return Err(NbError::ValidationError {
656                reason: format!("plan effect `{key}` has no file content"),
657                location: None,
658            });
659        };
660        let abs = root.join(&key);
661        // Never write through an existing symlink leaf or ancestor.
662        refuse_symlink_ancestors(root, &key)?;
663        if let Ok(meta) = std::fs::symlink_metadata(&abs) {
664            if meta.file_type().is_symlink() {
665                return Err(NbError::UnsupportedStructure {
666                    reason: format!(
667                        "refusing to write through symlink `{key}`"
668                    ),
669                });
670            }
671            // A planned create racing an external same-path create (file or
672            // directory appearing after the snapshot): refuse instead of
673            // overwriting the external note or failing obscurely on a dir.
674            if created_contains && !snapshot_keys.contains(&key) {
675                return Err(NbError::PathCollision {
676                    path: key,
677                    plan_index: None,
678                });
679            }
680        }
681        if let Some(parent) = abs.parent() {
682            std::fs::create_dir_all(parent).map_err(|e| NbError::Io {
683                path: parent.to_path_buf(),
684                source: e.into(),
685            })?;
686        }
687        std::fs::write(&abs, bytes).map_err(|e| NbError::Io {
688            path: abs,
689            source: e.into(),
690        })?;
691    }
692    Ok(())
693}