Skip to main content

nb_api/transaction/
commit.rs

1use std::collections::HashSet;
2use std::fs::OpenOptions;
3use std::path::{Path, PathBuf};
4
5use crate::error::NbError;
6use crate::gate;
7use crate::git;
8use crate::types::{CommitOutcome, OpOutcome};
9
10use super::apply::{OpMeta, PlanEffects, validate_and_apply_virtual};
11use super::index::{
12    PendingIndexEdit, acquire_index_lock, apply_index_edits, index_id_on_disk,
13    index_rel_for_folder, is_index_rel, numeric_selector,
14    reap_stale_index_lock, split_folder_basename,
15};
16use super::plan::annotate_plan_index;
17use super::virtual_tree::{
18    VirtualNode, VirtualTree, normalize_rel, parent_path,
19    refuse_symlink_ancestors,
20};
21
22use super::plan::Transaction;
23
24impl Transaction {
25    /// Validate-all, apply-all, at most one Git checkpoint.
26    pub async fn commit(self) -> Result<CommitOutcome, NbError> {
27        // Resolve root without invoking `nb` when possible (avoids nb
28        // auto-checkpoint clearing dirty state before the baseline check).
29        let notebook_root = self
30            .client
31            .show_notebook_path_unguarded(Some(&self.notebook))
32            .await?;
33        let gate_key = gate::git_common_dir_realpath(&notebook_root)?;
34        let _hold =
35            gate::acquire_notebook(gate_key, self.gate_timeout, false).await?;
36
37        // A crashed writer may leave its `.nb-api-index.lock` behind; an
38        // untracked lock file reads as dirty, so reap a stale one first.
39        reap_stale_index_lock(&notebook_root, self.gate_timeout);
40
41        if git::notebook_is_dirty(&notebook_root)? {
42            // Index-only reconciliation droppings (e.g. an external
43            // `nb` background checkpoint racing our gate) are
44            // absorbed by our own checkpoint below; anything else
45            // still refuses. See `git::status_is_index_only`.
46            let status = git::notebook_status_porcelain(&notebook_root)?;
47            if !git::status_is_index_only(&status) {
48                return Err(NbError::DirtyBaseline {
49                    guidance: "commit or clean the notebook worktree/index before Transaction::commit"
50                        .into(),
51                });
52            }
53        }
54
55        let pre_revision = git::notebook_head(&notebook_root)?;
56        if self.plan.is_empty() {
57            return Ok(CommitOutcome {
58                commit_created: false,
59                revision_id: None,
60                pre_revision,
61                ops: Vec::new(),
62            });
63        }
64
65        let ignored_existing: HashSet<String> =
66            git::list_ignored_paths(&notebook_root)?
67                .into_iter()
68                .map(|p| normalize_rel(&p))
69                .collect();
70        let mut tree = VirtualTree::from_disk(&notebook_root)?;
71        let mut op_meta: Vec<OpMeta> = Vec::with_capacity(self.plan.len());
72        let mut effects = PlanEffects::default();
73        // Disk keys at snapshot time: materialize refuses a planned create
74        // that appears on disk afterwards (external same-path race).
75        let snapshot_keys: HashSet<String> =
76            tree.nodes.keys().cloned().collect();
77
78        for (index, op) in self.plan.iter().enumerate() {
79            match validate_and_apply_virtual(
80                &self.notebook,
81                &notebook_root,
82                &mut tree,
83                &ignored_existing,
84                op,
85                index,
86                &mut effects,
87            ) {
88                Ok(meta) => op_meta.push(meta),
89                Err(err) => {
90                    return Err(annotate_plan_index(err, index as u32));
91                },
92            }
93        }
94
95        #[cfg(feature = "testing")]
96        if std::env::var_os("NB_API_SIMULATE_EXTERNAL_WRITER").is_some() {
97            // Deterministic stand-in for an external `nb add` racing our
98            // snapshot: create AND commit a note this transaction knows
99            // nothing about. Our commit must preserve it (file + index line).
100            simulate_external_writer(&notebook_root)?;
101        }
102
103        // Disk baseline before materialize (for rollback of new owned outputs).
104        // Virtual tree already includes planned ops; disk is still pre-apply.
105        let baseline_on_disk: HashSet<String> = {
106            let mut s: HashSet<String> =
107                git::list_notebook_paths(&notebook_root)?
108                    .into_iter()
109                    .map(|p| normalize_rel(&p))
110                    .collect();
111            s.extend(ignored_existing.iter().cloned());
112            s
113        };
114
115        // Force-stage every final file path so ignore rules cannot drop
116        // transaction-owned outputs (new ignored names, `.gitkeep`, etc.).
117        // `.index` files are applied post-materialize (fresh read under the
118        // index lock) but staged by the same checkpoint, so list them too.
119        let mut force_paths: Vec<String> = tree
120            .nodes
121            .iter()
122            .filter_map(|(p, n)| match n {
123                VirtualNode::File(_) => Some(p.clone()),
124                VirtualNode::Folder => None,
125            })
126            .collect();
127        for meta in &op_meta {
128            for edit in &meta.index_edits {
129                if !force_paths.iter().any(|p| p == &edit.file_rel) {
130                    force_paths.push(edit.file_rel.clone());
131                }
132            }
133        }
134
135        // Directories that already exist before materialize must never be
136        // pruned during rollback (including empty ignored parents).
137        let baseline_dirs = existing_dirs_under(&notebook_root);
138
139        // Materialize non-index files, apply `.index` edits from fresh
140        // on-disk content under `.nb-api-index.lock` (released before git
141        // staging — never held across `git add`/`commit`), then checkpoint.
142        // The lock serializes nb-api writers only; external `nb` writers are
143        // covered by O_APPEND plus post-write selector derivation.
144        let apply_result = async {
145            materialize_tree(&notebook_root, &tree, &effects, &snapshot_keys)?;
146            let mut pending: Vec<(u32, PendingIndexEdit)> = Vec::new();
147            for (i, meta) in op_meta.iter().enumerate() {
148                for edit in &meta.index_edits {
149                    pending.push((i as u32, edit.clone()));
150                }
151            }
152            let placements = if pending.is_empty() {
153                Vec::new()
154            } else {
155                let _lock =
156                    acquire_index_lock(&notebook_root, self.gate_timeout)
157                        .await?;
158                let placements = apply_index_edits(
159                    &notebook_root,
160                    &pending,
161                    self.gate_timeout,
162                )?;
163                drop(_lock);
164                placements
165            };
166            let created = git::notebook_commit_all(
167                &notebook_root,
168                &format!("nb-api transaction ({} ops)", self.plan.len()),
169                &force_paths,
170            )?;
171            Ok::<_, NbError>((created, placements))
172        }
173        .await;
174
175        match apply_result {
176            Ok((created, placements)) => {
177                let revision_id = if created {
178                    match git::notebook_head(&notebook_root) {
179                        Ok(head) => Some(head),
180                        Err(_) => {
181                            return Err(NbError::IndeterminateCommit {
182                                pre_revision,
183                                post_revision_observed: None,
184                                guidance: "git commit may have succeeded but HEAD could not be re-read; do not retry blindly".into(),
185                            });
186                        },
187                    }
188                } else {
189                    None
190                };
191                let ops = op_meta
192                    .into_iter()
193                    .enumerate()
194                    .map(|(i, meta)| {
195                        // Creates/moves/deletes carry post-write placements:
196                        // selector becomes the numeric `<folder>/<id>` form.
197                        // A cross-folder move records both a blank (source)
198                        // and an append (destination); the destination
199                        // placement is last, so take the last match.
200                        // All other ops keep the echoed target selector and
201                        // gain a numeric id when the path is indexed.
202                        let (selector, numeric_id) = match placements
203                            .iter()
204                            .rev()
205                            .find(|p| p.op_index == i as u32)
206                        {
207                            Some(p) => (
208                                Some(numeric_selector(
209                                    &self.notebook,
210                                    &p.folder,
211                                    p.id,
212                                )),
213                                Some(p.id),
214                            ),
215                            None => (
216                                meta.selector,
217                                meta.path.as_deref().and_then(|p| {
218                                    index_id_on_disk(&notebook_root, p)
219                                }),
220                            ),
221                        };
222                        OpOutcome {
223                            index: i as u32,
224                            path: meta.path,
225                            selector,
226                            numeric_id,
227                            noop: meta.noop,
228                            fingerprint: meta.fingerprint,
229                        }
230                    })
231                    .collect();
232                Ok(CommitOutcome {
233                    commit_created: created,
234                    revision_id,
235                    pre_revision,
236                    ops,
237                })
238            },
239            Err(err) => {
240                let post = git::notebook_head(&notebook_root).ok();
241                if post.as_deref().is_some_and(|h| h != pre_revision.as_str())
242                {
243                    return Err(NbError::IndeterminateCommit {
244                        pre_revision,
245                        post_revision_observed: post,
246                        guidance: "checkpoint may have completed; re-read HEAD/status and do not retry the same plan blindly".into(),
247                    });
248                }
249                // Owned outputs that did not exist at baseline must be removed
250                // even when ignored (git clean -fd keeps ignored files).
251                // Baseline directories are never owned outputs: pruning one
252                // would `remove_dir_all` a pre-existing folder.
253                let owned_new: Vec<String> = force_paths
254                    .into_iter()
255                    .filter(|p| !baseline_on_disk.contains(p))
256                    .filter(|p| !baseline_dirs.contains(p))
257                    .collect();
258                match try_restore(
259                    &notebook_root,
260                    &pre_revision,
261                    &owned_new,
262                    &baseline_dirs,
263                ) {
264                    Ok(()) => Err(err),
265                    Err(recovery) => Err(recovery),
266                }
267            },
268        }
269    }
270}
271
272/// All directory paths under the notebook root before materialize (relative,
273/// `/`-separated). Used so rollback never deletes pre-existing dirs.
274fn existing_dirs_under(notebook_root: &Path) -> HashSet<String> {
275    let mut dirs = HashSet::new();
276    fn walk(base: &Path, rel: &Path, out: &mut HashSet<String>) {
277        let entries = match std::fs::read_dir(base) {
278            Ok(e) => e,
279            Err(_) => return,
280        };
281        for entry in entries.flatten() {
282            let name = entry.file_name();
283            if name == ".git" {
284                continue;
285            }
286            let Ok(ft) = entry.file_type() else {
287                continue;
288            };
289            // Do not follow symlinks.
290            if ft.is_symlink() || !ft.is_dir() {
291                continue;
292            }
293            let child_rel = if rel.as_os_str().is_empty() {
294                PathBuf::from(&name)
295            } else {
296                rel.join(&name)
297            };
298            let key = child_rel.to_string_lossy().replace('\\', "/");
299            out.insert(key);
300            walk(&entry.path(), &child_rel, out);
301        }
302    }
303    walk(notebook_root, Path::new(""), &mut dirs);
304    dirs
305}
306
307fn try_restore(
308    notebook_root: &Path,
309    pre_revision: &str,
310    remove_new_owned: &[String],
311    baseline_dirs: &HashSet<String>,
312) -> Result<(), NbError> {
313    if let Err(e) = git::notebook_reset_clean(notebook_root, pre_revision) {
314        let post = git::notebook_head(notebook_root).ok();
315        let status = git::notebook_status_porcelain(notebook_root).ok();
316        return Err(NbError::RecoveryRequired {
317            pre_revision: pre_revision.to_string(),
318            post_revision_observed: post,
319            status_observed: status,
320            preserved_paths: Some(remove_new_owned.to_vec()),
321            guidance: format!(
322                "cleanup after failed commit could not be verified; inspect HEAD/status before retry. underlying: {e}"
323            ),
324        });
325    }
326    // `git clean -fd` can remove empty untracked dirs (including ignored
327    // parents) once staged children are cleared by reset --hard. Restore any
328    // baseline directory that disappeared.
329    for d in baseline_dirs {
330        let abs = notebook_root.join(d);
331        if !abs.exists()
332            && let Err(e) = std::fs::create_dir_all(&abs)
333        {
334            return Err(NbError::RecoveryRequired {
335                pre_revision: pre_revision.to_string(),
336                post_revision_observed: git::notebook_head(notebook_root).ok(),
337                status_observed: git::notebook_status_porcelain(notebook_root)
338                    .ok(),
339                preserved_paths: Some(vec![d.clone()]),
340                guidance: format!(
341                    "failed to restore baseline directory `{d}` after cleanup; do not retry blindly. underlying: {e}"
342                ),
343            });
344        }
345    }
346    // Explicitly delete transaction-owned outputs that reset/clean leave behind
347    // when they are Git-ignored untracked files.
348    let mut remaining = Vec::new();
349    for rel in remove_new_owned {
350        let abs = notebook_root.join(rel);
351        match std::fs::symlink_metadata(&abs) {
352            Ok(meta) if meta.file_type().is_symlink() || meta.is_file() => {
353                if let Err(e) = std::fs::remove_file(&abs) {
354                    remaining.push(format!("{rel} ({e})"));
355                }
356            },
357            Ok(meta) if meta.is_dir() => {
358                if let Err(e) = std::fs::remove_dir_all(&abs) {
359                    remaining.push(format!("{rel} ({e})"));
360                }
361            },
362            Ok(_) => {},
363            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {},
364            Err(e) => remaining.push(format!("{rel} ({e})")),
365        }
366        // Prune only empty parents that the transaction created — never a
367        // directory that existed at baseline (e.g. pre-existing ignored parent).
368        if let Some(parent) = parent_path(rel) {
369            let mut cur = parent;
370            loop {
371                if baseline_dirs.contains(&cur) {
372                    break;
373                }
374                let p = notebook_root.join(&cur);
375                match std::fs::remove_dir(&p) {
376                    Ok(()) => {},
377                    Err(_) => break,
378                }
379                match parent_path(&cur) {
380                    Some(next) => cur = next,
381                    None => break,
382                }
383            }
384        }
385    }
386    if !remaining.is_empty() {
387        return Err(NbError::RecoveryRequired {
388            pre_revision: pre_revision.to_string(),
389            post_revision_observed: git::notebook_head(notebook_root).ok(),
390            status_observed: git::notebook_status_porcelain(notebook_root).ok(),
391            preserved_paths: Some(remaining),
392            guidance: "failed to remove transaction-owned outputs after aborted commit; do not retry blindly".into(),
393        });
394    }
395    // Verify none of the new owned paths remain (including ignored).
396    let mut still_present = Vec::new();
397    for rel in remove_new_owned {
398        if notebook_root.join(rel).exists() {
399            still_present.push(rel.clone());
400        }
401    }
402    let head = match restore_verify_head(notebook_root) {
403        Ok(h) => h,
404        Err(e) => {
405            return Err(NbError::RecoveryRequired {
406                pre_revision: pre_revision.to_string(),
407                post_revision_observed: None,
408                status_observed: git::notebook_status_porcelain(notebook_root)
409                    .ok(),
410                preserved_paths: if still_present.is_empty() {
411                    None
412                } else {
413                    Some(still_present)
414                },
415                guidance: format!(
416                    "cleanup ran but HEAD re-read failed during verification; do not retry blindly. underlying: {e}"
417                ),
418            });
419        },
420    };
421    let dirty = match restore_verify_dirty(notebook_root) {
422        Ok(d) => d,
423        Err(e) => {
424            return Err(NbError::RecoveryRequired {
425                pre_revision: pre_revision.to_string(),
426                post_revision_observed: Some(head),
427                status_observed: git::notebook_status_porcelain(notebook_root)
428                    .ok(),
429                preserved_paths: if still_present.is_empty() {
430                    None
431                } else {
432                    Some(still_present)
433                },
434                guidance: format!(
435                    "cleanup ran but dirty-status check failed during verification; do not retry blindly. underlying: {e}"
436                ),
437            });
438        },
439    };
440    if head != pre_revision || dirty || !still_present.is_empty() {
441        return Err(NbError::RecoveryRequired {
442            pre_revision: pre_revision.to_string(),
443            post_revision_observed: Some(head),
444            status_observed: git::notebook_status_porcelain(notebook_root).ok(),
445            preserved_paths: if still_present.is_empty() {
446                None
447            } else {
448                Some(still_present)
449            },
450            guidance:
451                "cleanup ran but HEAD/status/owned-path verification failed; do not retry blindly"
452                    .into(),
453        });
454    }
455    Ok(())
456}
457
458/// HEAD observation used only in post-cleanup verification.
459fn restore_verify_head(notebook_root: &Path) -> Result<String, NbError> {
460    #[cfg(feature = "testing")]
461    if std::env::var_os("NB_API_FAIL_RESTORE_HEAD").is_some() {
462        return Err(NbError::CommandFailed {
463            command: "nb-api://fail-restore-head".into(),
464            stderr: "injected HEAD verify failure for rollback tests".into(),
465            exit_code: Some(1),
466        });
467    }
468    git::notebook_head(notebook_root)
469}
470
471/// Dirty-status observation used only in post-cleanup verification.
472fn restore_verify_dirty(notebook_root: &Path) -> Result<bool, NbError> {
473    #[cfg(feature = "testing")]
474    if std::env::var_os("NB_API_FAIL_RESTORE_DIRTY").is_some() {
475        return Err(NbError::CommandFailed {
476            command: "nb-api://fail-restore-dirty".into(),
477            stderr: "injected dirty-status verify failure for rollback tests"
478                .into(),
479            exit_code: Some(1),
480        });
481    }
482    git::notebook_is_dirty(notebook_root)
483}
484
485/// Deterministic stand-in for an external `nb add` racing our snapshot
486/// (testing only, behind `NB_API_SIMULATE_EXTERNAL_WRITER`): create AND
487/// commit a note plus its `.index` line that this transaction knows nothing
488/// about. The commit must preserve both.
489///
490/// `NB_API_SIMULATE_EXTERNAL_SAME_PATH=<rel>` makes the external writer take
491/// the given path instead (to test the same-path race: our commit must fail
492/// `PathCollision`, never overwrite).
493#[cfg(feature = "testing")]
494fn simulate_external_writer(notebook_root: &Path) -> Result<(), NbError> {
495    use std::io::Write as _;
496    let (folder, name) =
497        match std::env::var("NB_API_SIMULATE_EXTERNAL_SAME_PATH") {
498            Ok(rel) => {
499                let (folder, basename) =
500                    split_folder_basename(&normalize_rel(&rel));
501                (folder, basename)
502            },
503            Err(_) => {
504                (String::new(), format!("external-{}.md", std::process::id()))
505            },
506        };
507    let rel = if folder.is_empty() {
508        name.clone()
509    } else {
510        format!("{folder}/{name}")
511    };
512    if !folder.is_empty() {
513        std::fs::create_dir_all(notebook_root.join(&folder)).map_err(|e| {
514            NbError::Io {
515                path: notebook_root.join(&folder),
516                source: e.into(),
517            }
518        })?;
519    }
520    std::fs::write(notebook_root.join(&rel), b"# External\n\nexternal body\n")
521        .map_err(|e| NbError::Io {
522            path: notebook_root.join(&rel),
523            source: e.into(),
524        })?;
525    let index_abs = notebook_root.join(index_rel_for_folder(&folder));
526    let mut f = OpenOptions::new()
527        .create(true)
528        .append(true)
529        .open(&index_abs)
530        .map_err(|e| NbError::Io {
531            path: index_abs.clone(),
532            source: e.into(),
533        })?;
534    writeln!(f, "{name}").map_err(|e| NbError::Io {
535        path: index_abs.clone(),
536        source: e.into(),
537    })?;
538    drop(f);
539    git::notebook_commit_all(
540        notebook_root,
541        "external writer",
542        &[rel, index_rel_for_folder(&folder)],
543    )?;
544    Ok(())
545}
546
547/// Write the plan's effects to disk.
548///
549/// ONLY paths recorded in `effects` are touched: files created after the
550/// snapshot by an external writer are never deleted, and files deleted
551/// externally are never resurrected. `.index` rels are skipped here (applied
552/// separately from fresh on-disk content under `.nb-api-index.lock`).
553///
554/// `snapshot_keys` are the tree keys at snapshot time. A `created` path that
555/// exists on disk now but was absent from the snapshot means an external
556/// writer created the same path after our collision validation — refuse with
557/// `PathCollision` instead of overwriting it (which would also leave
558/// ambiguous duplicate `.index` entries).
559fn materialize_tree(
560    root: &Path,
561    tree: &VirtualTree,
562    effects: &PlanEffects,
563    snapshot_keys: &HashSet<String>,
564) -> Result<(), NbError> {
565    // Preflight: no path may traverse an existing symlink ancestor (including
566    // ignored directory symlinks excluded from the virtual snapshot).
567    for rel in tree.nodes.keys() {
568        refuse_symlink_ancestors(root, rel)?;
569    }
570
571    // Remove ONLY plan delete/move sources that exist on disk as files.
572    let mut removed: Vec<&String> = effects.removed.iter().collect();
573    removed.sort();
574    removed.dedup();
575    for rel in removed {
576        let key = normalize_rel(rel);
577        let abs = root.join(&key);
578        let meta = match std::fs::symlink_metadata(&abs) {
579            Ok(m) => m,
580            Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
581            Err(e) => {
582                return Err(NbError::Io {
583                    path: abs,
584                    source: e.into(),
585                });
586            },
587        };
588        if meta.file_type().is_symlink() {
589            return Err(NbError::UnsupportedStructure {
590                reason: format!(
591                    "refusing to materialize through symlink `{key}`"
592                ),
593            });
594        }
595        if meta.is_dir() {
596            // Plan folder-delete source: remove the whole subtree.
597            // (Move sources are always files; folder moves are
598            // refused at plan validation.)
599            std::fs::remove_dir_all(&abs).map_err(|e| NbError::Io {
600                path: abs,
601                source: e.into(),
602            })?;
603            continue;
604        }
605        if meta.is_file()
606            && !matches!(tree.nodes.get(&key), Some(VirtualNode::File(_)))
607        {
608            std::fs::remove_file(&abs).map_err(|e| NbError::Io {
609                path: abs,
610                source: e.into(),
611            })?;
612        }
613    }
614    // Create ONLY plan folders.
615    let mut mkdirs: Vec<&String> = effects.mkdirs.iter().collect();
616    mkdirs.sort();
617    mkdirs.dedup();
618    for rel in mkdirs {
619        let key = normalize_rel(rel);
620        let abs = root.join(&key);
621        refuse_symlink_ancestors(root, &key)?;
622        std::fs::create_dir_all(&abs).map_err(|e| NbError::Io {
623            path: abs.clone(),
624            source: e.into(),
625        })?;
626    }
627    // Write ONLY plan-created/modified files (never `.index`).
628    let mut written: Vec<&String> = effects
629        .written
630        .iter()
631        .filter(|rel| !is_index_rel(rel))
632        .collect();
633    written.sort();
634    written.dedup();
635    let created: HashSet<&String> = effects.created.iter().collect();
636    for rel in written {
637        let key = normalize_rel(rel);
638        let created_contains = created.contains(rel);
639        let Some(VirtualNode::File(bytes)) = tree.nodes.get(&key) else {
640            return Err(NbError::ValidationError {
641                reason: format!("plan effect `{key}` has no file content"),
642                location: None,
643            });
644        };
645        let abs = root.join(&key);
646        // Never write through an existing symlink leaf or ancestor.
647        refuse_symlink_ancestors(root, &key)?;
648        if let Ok(meta) = std::fs::symlink_metadata(&abs) {
649            if meta.file_type().is_symlink() {
650                return Err(NbError::UnsupportedStructure {
651                    reason: format!(
652                        "refusing to write through symlink `{key}`"
653                    ),
654                });
655            }
656            // A planned create racing an external same-path create (file or
657            // directory appearing after the snapshot): refuse instead of
658            // overwriting the external note or failing obscurely on a dir.
659            if created_contains && !snapshot_keys.contains(&key) {
660                return Err(NbError::PathCollision {
661                    path: key,
662                    plan_index: None,
663                });
664            }
665        }
666        if let Some(parent) = abs.parent() {
667            std::fs::create_dir_all(parent).map_err(|e| NbError::Io {
668                path: parent.to_path_buf(),
669                source: e.into(),
670            })?;
671        }
672        std::fs::write(&abs, bytes).map_err(|e| NbError::Io {
673            path: abs,
674            source: e.into(),
675        })?;
676    }
677    Ok(())
678}