Skip to main content

nb_api/transaction/
commit.rs

1use std::collections::HashSet;
2use std::fs::OpenOptions;
3use std::path::{Path, PathBuf};
4
5use crate::error::NbError;
6use crate::gate;
7use crate::git;
8use crate::types::{CommitOutcome, OpOutcome};
9
10use super::apply::{OpMeta, PlanEffects, validate_and_apply_virtual};
11use super::index::{
12    PendingIndexEdit, acquire_index_lock, apply_index_edits, index_id_on_disk,
13    index_rel_for_folder, is_index_rel, numeric_selector,
14    reap_stale_index_lock, split_folder_basename,
15};
16use super::plan::annotate_plan_index;
17use super::virtual_tree::{
18    VirtualNode, VirtualTree, normalize_rel, parent_path,
19    refuse_symlink_ancestors,
20};
21
22use super::plan::Transaction;
23
24impl Transaction {
25    /// Validate-all, apply-all, at most one Git checkpoint.
26    pub async fn commit(self) -> Result<CommitOutcome, NbError> {
27        // Resolve root without invoking `nb` when possible (avoids nb
28        // auto-checkpoint clearing dirty state before the baseline check).
29        let notebook_root = self
30            .client
31            .show_notebook_path_unguarded(Some(&self.notebook))
32            .await?;
33        let gate_key = gate::git_common_dir_realpath(&notebook_root)?;
34        let _hold =
35            gate::acquire_notebook(gate_key, self.gate_timeout, false).await?;
36
37        // A crashed writer may leave its `.nb-api-index.lock` behind; an
38        // untracked lock file reads as dirty, so reap a stale one first.
39        reap_stale_index_lock(&notebook_root, self.gate_timeout);
40
41        if git::notebook_is_dirty(&notebook_root)? {
42            return Err(NbError::DirtyBaseline {
43                guidance: "commit or clean the notebook worktree/index before Transaction::commit"
44                    .into(),
45            });
46        }
47
48        let pre_revision = git::notebook_head(&notebook_root)?;
49        if self.plan.is_empty() {
50            return Ok(CommitOutcome {
51                commit_created: false,
52                revision_id: None,
53                pre_revision,
54                ops: Vec::new(),
55            });
56        }
57
58        let ignored_existing: HashSet<String> =
59            git::list_ignored_paths(&notebook_root)?
60                .into_iter()
61                .map(|p| normalize_rel(&p))
62                .collect();
63        let mut tree = VirtualTree::from_disk(&notebook_root)?;
64        let mut op_meta: Vec<OpMeta> = Vec::with_capacity(self.plan.len());
65        let mut effects = PlanEffects::default();
66        // Disk keys at snapshot time: materialize refuses a planned create
67        // that appears on disk afterwards (external same-path race).
68        let snapshot_keys: HashSet<String> =
69            tree.nodes.keys().cloned().collect();
70
71        for (index, op) in self.plan.iter().enumerate() {
72            match validate_and_apply_virtual(
73                &self.notebook,
74                &notebook_root,
75                &mut tree,
76                &ignored_existing,
77                op,
78                index,
79                &mut effects,
80            ) {
81                Ok(meta) => op_meta.push(meta),
82                Err(err) => {
83                    return Err(annotate_plan_index(err, index as u32));
84                },
85            }
86        }
87
88        #[cfg(feature = "testing")]
89        if std::env::var_os("NB_API_SIMULATE_EXTERNAL_WRITER").is_some() {
90            // Deterministic stand-in for an external `nb add` racing our
91            // snapshot: create AND commit a note this transaction knows
92            // nothing about. Our commit must preserve it (file + index line).
93            simulate_external_writer(&notebook_root)?;
94        }
95
96        // Disk baseline before materialize (for rollback of new owned outputs).
97        // Virtual tree already includes planned ops; disk is still pre-apply.
98        let baseline_on_disk: HashSet<String> = {
99            let mut s: HashSet<String> =
100                git::list_notebook_paths(&notebook_root)?
101                    .into_iter()
102                    .map(|p| normalize_rel(&p))
103                    .collect();
104            s.extend(ignored_existing.iter().cloned());
105            s
106        };
107
108        // Force-stage every final file path so ignore rules cannot drop
109        // transaction-owned outputs (new ignored names, `.gitkeep`, etc.).
110        // `.index` files are applied post-materialize (fresh read under the
111        // index lock) but staged by the same checkpoint, so list them too.
112        let mut force_paths: Vec<String> = tree
113            .nodes
114            .iter()
115            .filter_map(|(p, n)| match n {
116                VirtualNode::File(_) => Some(p.clone()),
117                VirtualNode::Folder => None,
118            })
119            .collect();
120        for meta in &op_meta {
121            for edit in &meta.index_edits {
122                if !force_paths.iter().any(|p| p == &edit.file_rel) {
123                    force_paths.push(edit.file_rel.clone());
124                }
125            }
126        }
127
128        // Directories that already exist before materialize must never be
129        // pruned during rollback (including empty ignored parents).
130        let baseline_dirs = existing_dirs_under(&notebook_root);
131
132        // Materialize non-index files, apply `.index` edits from fresh
133        // on-disk content under `.nb-api-index.lock` (released before git
134        // staging — never held across `git add`/`commit`), then checkpoint.
135        // The lock serializes nb-api writers only; external `nb` writers are
136        // covered by O_APPEND plus post-write selector derivation.
137        let apply_result = async {
138            materialize_tree(&notebook_root, &tree, &effects, &snapshot_keys)?;
139            let mut pending: Vec<(u32, PendingIndexEdit)> = Vec::new();
140            for (i, meta) in op_meta.iter().enumerate() {
141                for edit in &meta.index_edits {
142                    pending.push((i as u32, edit.clone()));
143                }
144            }
145            let placements = if pending.is_empty() {
146                Vec::new()
147            } else {
148                let _lock =
149                    acquire_index_lock(&notebook_root, self.gate_timeout)
150                        .await?;
151                let placements = apply_index_edits(
152                    &notebook_root,
153                    &pending,
154                    self.gate_timeout,
155                )?;
156                drop(_lock);
157                placements
158            };
159            let created = git::notebook_commit_all(
160                &notebook_root,
161                &format!("nb-api transaction ({} ops)", self.plan.len()),
162                &force_paths,
163            )?;
164            Ok::<_, NbError>((created, placements))
165        }
166        .await;
167
168        match apply_result {
169            Ok((created, placements)) => {
170                let revision_id = if created {
171                    match git::notebook_head(&notebook_root) {
172                        Ok(head) => Some(head),
173                        Err(_) => {
174                            return Err(NbError::IndeterminateCommit {
175                                pre_revision,
176                                post_revision_observed: None,
177                                guidance: "git commit may have succeeded but HEAD could not be re-read; do not retry blindly".into(),
178                            });
179                        },
180                    }
181                } else {
182                    None
183                };
184                let ops = op_meta
185                    .into_iter()
186                    .enumerate()
187                    .map(|(i, meta)| {
188                        // Creates/moves/deletes carry post-write placements:
189                        // selector becomes the numeric `<folder>/<id>` form.
190                        // A cross-folder move records both a blank (source)
191                        // and an append (destination); the destination
192                        // placement is last, so take the last match.
193                        // All other ops keep the echoed target selector and
194                        // gain a numeric id when the path is indexed.
195                        let (selector, numeric_id) = match placements
196                            .iter()
197                            .rev()
198                            .find(|p| p.op_index == i as u32)
199                        {
200                            Some(p) => (
201                                Some(numeric_selector(
202                                    &self.notebook,
203                                    &p.folder,
204                                    p.id,
205                                )),
206                                Some(p.id),
207                            ),
208                            None => (
209                                meta.selector,
210                                meta.path.as_deref().and_then(|p| {
211                                    index_id_on_disk(&notebook_root, p)
212                                }),
213                            ),
214                        };
215                        OpOutcome {
216                            index: i as u32,
217                            path: meta.path,
218                            selector,
219                            numeric_id,
220                            noop: meta.noop,
221                            fingerprint: meta.fingerprint,
222                        }
223                    })
224                    .collect();
225                Ok(CommitOutcome {
226                    commit_created: created,
227                    revision_id,
228                    pre_revision,
229                    ops,
230                })
231            },
232            Err(err) => {
233                let post = git::notebook_head(&notebook_root).ok();
234                if post.as_deref().is_some_and(|h| h != pre_revision.as_str())
235                {
236                    return Err(NbError::IndeterminateCommit {
237                        pre_revision,
238                        post_revision_observed: post,
239                        guidance: "checkpoint may have completed; re-read HEAD/status and do not retry the same plan blindly".into(),
240                    });
241                }
242                // Owned outputs that did not exist at baseline must be removed
243                // even when ignored (git clean -fd keeps ignored files).
244                // Baseline directories are never owned outputs: pruning one
245                // would `remove_dir_all` a pre-existing folder.
246                let owned_new: Vec<String> = force_paths
247                    .into_iter()
248                    .filter(|p| !baseline_on_disk.contains(p))
249                    .filter(|p| !baseline_dirs.contains(p))
250                    .collect();
251                match try_restore(
252                    &notebook_root,
253                    &pre_revision,
254                    &owned_new,
255                    &baseline_dirs,
256                ) {
257                    Ok(()) => Err(err),
258                    Err(recovery) => Err(recovery),
259                }
260            },
261        }
262    }
263}
264
265/// All directory paths under the notebook root before materialize (relative,
266/// `/`-separated). Used so rollback never deletes pre-existing dirs.
267fn existing_dirs_under(notebook_root: &Path) -> HashSet<String> {
268    let mut dirs = HashSet::new();
269    fn walk(base: &Path, rel: &Path, out: &mut HashSet<String>) {
270        let entries = match std::fs::read_dir(base) {
271            Ok(e) => e,
272            Err(_) => return,
273        };
274        for entry in entries.flatten() {
275            let name = entry.file_name();
276            if name == ".git" {
277                continue;
278            }
279            let Ok(ft) = entry.file_type() else {
280                continue;
281            };
282            // Do not follow symlinks.
283            if ft.is_symlink() || !ft.is_dir() {
284                continue;
285            }
286            let child_rel = if rel.as_os_str().is_empty() {
287                PathBuf::from(&name)
288            } else {
289                rel.join(&name)
290            };
291            let key = child_rel.to_string_lossy().replace('\\', "/");
292            out.insert(key);
293            walk(&entry.path(), &child_rel, out);
294        }
295    }
296    walk(notebook_root, Path::new(""), &mut dirs);
297    dirs
298}
299
300fn try_restore(
301    notebook_root: &Path,
302    pre_revision: &str,
303    remove_new_owned: &[String],
304    baseline_dirs: &HashSet<String>,
305) -> Result<(), NbError> {
306    if let Err(e) = git::notebook_reset_clean(notebook_root, pre_revision) {
307        let post = git::notebook_head(notebook_root).ok();
308        let status = git::notebook_status_porcelain(notebook_root).ok();
309        return Err(NbError::RecoveryRequired {
310            pre_revision: pre_revision.to_string(),
311            post_revision_observed: post,
312            status_observed: status,
313            preserved_paths: Some(remove_new_owned.to_vec()),
314            guidance: format!(
315                "cleanup after failed commit could not be verified; inspect HEAD/status before retry. underlying: {e}"
316            ),
317        });
318    }
319    // `git clean -fd` can remove empty untracked dirs (including ignored
320    // parents) once staged children are cleared by reset --hard. Restore any
321    // baseline directory that disappeared.
322    for d in baseline_dirs {
323        let abs = notebook_root.join(d);
324        if !abs.exists()
325            && let Err(e) = std::fs::create_dir_all(&abs)
326        {
327            return Err(NbError::RecoveryRequired {
328                pre_revision: pre_revision.to_string(),
329                post_revision_observed: git::notebook_head(notebook_root).ok(),
330                status_observed: git::notebook_status_porcelain(notebook_root)
331                    .ok(),
332                preserved_paths: Some(vec![d.clone()]),
333                guidance: format!(
334                    "failed to restore baseline directory `{d}` after cleanup; do not retry blindly. underlying: {e}"
335                ),
336            });
337        }
338    }
339    // Explicitly delete transaction-owned outputs that reset/clean leave behind
340    // when they are Git-ignored untracked files.
341    let mut remaining = Vec::new();
342    for rel in remove_new_owned {
343        let abs = notebook_root.join(rel);
344        match std::fs::symlink_metadata(&abs) {
345            Ok(meta) if meta.file_type().is_symlink() || meta.is_file() => {
346                if let Err(e) = std::fs::remove_file(&abs) {
347                    remaining.push(format!("{rel} ({e})"));
348                }
349            },
350            Ok(meta) if meta.is_dir() => {
351                if let Err(e) = std::fs::remove_dir_all(&abs) {
352                    remaining.push(format!("{rel} ({e})"));
353                }
354            },
355            Ok(_) => {},
356            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {},
357            Err(e) => remaining.push(format!("{rel} ({e})")),
358        }
359        // Prune only empty parents that the transaction created — never a
360        // directory that existed at baseline (e.g. pre-existing ignored parent).
361        if let Some(parent) = parent_path(rel) {
362            let mut cur = parent;
363            loop {
364                if baseline_dirs.contains(&cur) {
365                    break;
366                }
367                let p = notebook_root.join(&cur);
368                match std::fs::remove_dir(&p) {
369                    Ok(()) => {},
370                    Err(_) => break,
371                }
372                match parent_path(&cur) {
373                    Some(next) => cur = next,
374                    None => break,
375                }
376            }
377        }
378    }
379    if !remaining.is_empty() {
380        return Err(NbError::RecoveryRequired {
381            pre_revision: pre_revision.to_string(),
382            post_revision_observed: git::notebook_head(notebook_root).ok(),
383            status_observed: git::notebook_status_porcelain(notebook_root).ok(),
384            preserved_paths: Some(remaining),
385            guidance: "failed to remove transaction-owned outputs after aborted commit; do not retry blindly".into(),
386        });
387    }
388    // Verify none of the new owned paths remain (including ignored).
389    let mut still_present = Vec::new();
390    for rel in remove_new_owned {
391        if notebook_root.join(rel).exists() {
392            still_present.push(rel.clone());
393        }
394    }
395    let head = match restore_verify_head(notebook_root) {
396        Ok(h) => h,
397        Err(e) => {
398            return Err(NbError::RecoveryRequired {
399                pre_revision: pre_revision.to_string(),
400                post_revision_observed: None,
401                status_observed: git::notebook_status_porcelain(notebook_root)
402                    .ok(),
403                preserved_paths: if still_present.is_empty() {
404                    None
405                } else {
406                    Some(still_present)
407                },
408                guidance: format!(
409                    "cleanup ran but HEAD re-read failed during verification; do not retry blindly. underlying: {e}"
410                ),
411            });
412        },
413    };
414    let dirty = match restore_verify_dirty(notebook_root) {
415        Ok(d) => d,
416        Err(e) => {
417            return Err(NbError::RecoveryRequired {
418                pre_revision: pre_revision.to_string(),
419                post_revision_observed: Some(head),
420                status_observed: git::notebook_status_porcelain(notebook_root)
421                    .ok(),
422                preserved_paths: if still_present.is_empty() {
423                    None
424                } else {
425                    Some(still_present)
426                },
427                guidance: format!(
428                    "cleanup ran but dirty-status check failed during verification; do not retry blindly. underlying: {e}"
429                ),
430            });
431        },
432    };
433    if head != pre_revision || dirty || !still_present.is_empty() {
434        return Err(NbError::RecoveryRequired {
435            pre_revision: pre_revision.to_string(),
436            post_revision_observed: Some(head),
437            status_observed: git::notebook_status_porcelain(notebook_root).ok(),
438            preserved_paths: if still_present.is_empty() {
439                None
440            } else {
441                Some(still_present)
442            },
443            guidance:
444                "cleanup ran but HEAD/status/owned-path verification failed; do not retry blindly"
445                    .into(),
446        });
447    }
448    Ok(())
449}
450
451/// HEAD observation used only in post-cleanup verification.
452fn restore_verify_head(notebook_root: &Path) -> Result<String, NbError> {
453    #[cfg(feature = "testing")]
454    if std::env::var_os("NB_API_FAIL_RESTORE_HEAD").is_some() {
455        return Err(NbError::CommandFailed {
456            command: "nb-api://fail-restore-head".into(),
457            stderr: "injected HEAD verify failure for rollback tests".into(),
458            exit_code: Some(1),
459        });
460    }
461    git::notebook_head(notebook_root)
462}
463
464/// Dirty-status observation used only in post-cleanup verification.
465fn restore_verify_dirty(notebook_root: &Path) -> Result<bool, NbError> {
466    #[cfg(feature = "testing")]
467    if std::env::var_os("NB_API_FAIL_RESTORE_DIRTY").is_some() {
468        return Err(NbError::CommandFailed {
469            command: "nb-api://fail-restore-dirty".into(),
470            stderr: "injected dirty-status verify failure for rollback tests"
471                .into(),
472            exit_code: Some(1),
473        });
474    }
475    git::notebook_is_dirty(notebook_root)
476}
477
478/// Deterministic stand-in for an external `nb add` racing our snapshot
479/// (testing only, behind `NB_API_SIMULATE_EXTERNAL_WRITER`): create AND
480/// commit a note plus its `.index` line that this transaction knows nothing
481/// about. The commit must preserve both.
482///
483/// `NB_API_SIMULATE_EXTERNAL_SAME_PATH=<rel>` makes the external writer take
484/// the given path instead (to test the same-path race: our commit must fail
485/// `PathCollision`, never overwrite).
486#[cfg(feature = "testing")]
487fn simulate_external_writer(notebook_root: &Path) -> Result<(), NbError> {
488    use std::io::Write as _;
489    let (folder, name) =
490        match std::env::var("NB_API_SIMULATE_EXTERNAL_SAME_PATH") {
491            Ok(rel) => {
492                let (folder, basename) =
493                    split_folder_basename(&normalize_rel(&rel));
494                (folder, basename)
495            },
496            Err(_) => {
497                (String::new(), format!("external-{}.md", std::process::id()))
498            },
499        };
500    let rel = if folder.is_empty() {
501        name.clone()
502    } else {
503        format!("{folder}/{name}")
504    };
505    if !folder.is_empty() {
506        std::fs::create_dir_all(notebook_root.join(&folder)).map_err(|e| {
507            NbError::Io {
508                path: notebook_root.join(&folder),
509                source: e.into(),
510            }
511        })?;
512    }
513    std::fs::write(notebook_root.join(&rel), b"# External\n\nexternal body\n")
514        .map_err(|e| NbError::Io {
515            path: notebook_root.join(&rel),
516            source: e.into(),
517        })?;
518    let index_abs = notebook_root.join(index_rel_for_folder(&folder));
519    let mut f = OpenOptions::new()
520        .create(true)
521        .append(true)
522        .open(&index_abs)
523        .map_err(|e| NbError::Io {
524            path: index_abs.clone(),
525            source: e.into(),
526        })?;
527    writeln!(f, "{name}").map_err(|e| NbError::Io {
528        path: index_abs.clone(),
529        source: e.into(),
530    })?;
531    drop(f);
532    git::notebook_commit_all(
533        notebook_root,
534        "external writer",
535        &[rel, index_rel_for_folder(&folder)],
536    )?;
537    Ok(())
538}
539
540/// Write the plan's effects to disk.
541///
542/// ONLY paths recorded in `effects` are touched: files created after the
543/// snapshot by an external writer are never deleted, and files deleted
544/// externally are never resurrected. `.index` rels are skipped here (applied
545/// separately from fresh on-disk content under `.nb-api-index.lock`).
546///
547/// `snapshot_keys` are the tree keys at snapshot time. A `created` path that
548/// exists on disk now but was absent from the snapshot means an external
549/// writer created the same path after our collision validation — refuse with
550/// `PathCollision` instead of overwriting it (which would also leave
551/// ambiguous duplicate `.index` entries).
552fn materialize_tree(
553    root: &Path,
554    tree: &VirtualTree,
555    effects: &PlanEffects,
556    snapshot_keys: &HashSet<String>,
557) -> Result<(), NbError> {
558    // Preflight: no path may traverse an existing symlink ancestor (including
559    // ignored directory symlinks excluded from the virtual snapshot).
560    for rel in tree.nodes.keys() {
561        refuse_symlink_ancestors(root, rel)?;
562    }
563
564    // Remove ONLY plan delete/move sources that exist on disk as files.
565    let mut removed: Vec<&String> = effects.removed.iter().collect();
566    removed.sort();
567    removed.dedup();
568    for rel in removed {
569        let key = normalize_rel(rel);
570        let abs = root.join(&key);
571        let meta = match std::fs::symlink_metadata(&abs) {
572            Ok(m) => m,
573            Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
574            Err(e) => {
575                return Err(NbError::Io {
576                    path: abs,
577                    source: e.into(),
578                });
579            },
580        };
581        if meta.file_type().is_symlink() {
582            return Err(NbError::UnsupportedStructure {
583                reason: format!(
584                    "refusing to materialize through symlink `{key}`"
585                ),
586            });
587        }
588        if meta.is_file()
589            && !matches!(tree.nodes.get(&key), Some(VirtualNode::File(_)))
590        {
591            std::fs::remove_file(&abs).map_err(|e| NbError::Io {
592                path: abs,
593                source: e.into(),
594            })?;
595        }
596    }
597    // Create ONLY plan folders.
598    let mut mkdirs: Vec<&String> = effects.mkdirs.iter().collect();
599    mkdirs.sort();
600    mkdirs.dedup();
601    for rel in mkdirs {
602        let key = normalize_rel(rel);
603        let abs = root.join(&key);
604        refuse_symlink_ancestors(root, &key)?;
605        std::fs::create_dir_all(&abs).map_err(|e| NbError::Io {
606            path: abs.clone(),
607            source: e.into(),
608        })?;
609    }
610    // Write ONLY plan-created/modified files (never `.index`).
611    let mut written: Vec<&String> = effects
612        .written
613        .iter()
614        .filter(|rel| !is_index_rel(rel))
615        .collect();
616    written.sort();
617    written.dedup();
618    let created: HashSet<&String> = effects.created.iter().collect();
619    for rel in written {
620        let key = normalize_rel(rel);
621        let created_contains = created.contains(rel);
622        let Some(VirtualNode::File(bytes)) = tree.nodes.get(&key) else {
623            return Err(NbError::ValidationError {
624                reason: format!("plan effect `{key}` has no file content"),
625                location: None,
626            });
627        };
628        let abs = root.join(&key);
629        // Never write through an existing symlink leaf or ancestor.
630        refuse_symlink_ancestors(root, &key)?;
631        if let Ok(meta) = std::fs::symlink_metadata(&abs) {
632            if meta.file_type().is_symlink() {
633                return Err(NbError::UnsupportedStructure {
634                    reason: format!(
635                        "refusing to write through symlink `{key}`"
636                    ),
637                });
638            }
639            // A planned create racing an external same-path create (file or
640            // directory appearing after the snapshot): refuse instead of
641            // overwriting the external note or failing obscurely on a dir.
642            if created_contains && !snapshot_keys.contains(&key) {
643                return Err(NbError::PathCollision {
644                    path: key,
645                    plan_index: None,
646                });
647            }
648        }
649        if let Some(parent) = abs.parent() {
650            std::fs::create_dir_all(parent).map_err(|e| NbError::Io {
651                path: parent.to_path_buf(),
652                source: e.into(),
653            })?;
654        }
655        std::fs::write(&abs, bytes).map_err(|e| NbError::Io {
656            path: abs,
657            source: e.into(),
658        })?;
659    }
660    Ok(())
661}