grit_lib/
fast_export.rs

1//! [`git fast-export`](https://git-scm.com/docs/git-fast-export) stream generation.
2//!
3//! Supports the subset needed by upstream tests: `--all`, `--anonymize`,
4//! `--anonymize-map`, topological commit order with `reverse` (oldest first),
5//! blob/commit marks, per-commit tree diffs, and annotated tags on commits.
6
7use std::collections::{HashMap, HashSet};
8use std::io::Write;
9
10use crate::diff::{diff_trees, DiffEntry, DiffStatus};
11use crate::error::{Error, Result};
12use crate::objects::{parse_commit, parse_tag, CommitData, ObjectId, ObjectKind};
13use crate::refs;
14use crate::repo::Repository;
15use crate::rev_list::{rev_list, OrderingMode, RevListOptions};
16
17use crate::index::{MODE_GITLINK, MODE_TREE};
18
19/// Options for [`export_stream`].
20#[derive(Debug, Clone, Default)]
21pub struct FastExportOptions {
22    /// Export all heads under `refs/heads/` (and reachable history).
23    pub all: bool,
24    /// Replace paths, idents, messages, and non-mark OIDs with stable placeholders.
25    pub anonymize: bool,
26    /// `from:to` or bare `token` mappings (last duplicate key wins, matching Git).
27    pub anonymize_maps: Vec<String>,
28    /// Emit `feature done` / trailing `done` (matches `git fast-import` when the feature is negotiated).
29    pub use_done_feature: bool,
30    /// Omit `blob` commands and emit `M` lines with full object ids (matches `git fast-export --no-data`).
31    pub no_data: bool,
32}
33
34struct AnonState<'a> {
35    seeds: &'a HashMap<String, String>,
36    paths: HashMap<String, String>,
37    refs: HashMap<String, String>,
38    objs: HashMap<String, String>,
39    idents: HashMap<String, String>,
40    tag_msgs: HashMap<String, String>,
41    path_n: u32,
42    ref_n: u32,
43    oid_n: u32,
44    ident_n: u32,
45    subject_n: u32,
46    tag_msg_n: u32,
47    blob_n: u32,
48}
49
50impl<'a> AnonState<'a> {
51    fn new(seeds: &'a HashMap<String, String>) -> Self {
52        Self {
53            seeds,
54            paths: HashMap::new(),
55            refs: HashMap::new(),
56            objs: HashMap::new(),
57            idents: HashMap::new(),
58            tag_msgs: HashMap::new(),
59            path_n: 0,
60            ref_n: 0,
61            oid_n: 0,
62            ident_n: 0,
63            subject_n: 0,
64            tag_msg_n: 0,
65            blob_n: 0,
66        }
67    }
68
69    fn map_token(
70        map: &mut HashMap<String, String>,
71        seeds: &HashMap<String, String>,
72        key: &str,
73        gen: impl FnOnce() -> String,
74    ) -> String {
75        if let Some(v) = seeds.get(key) {
76            return v.clone();
77        }
78        if let Some(v) = map.get(key) {
79            return v.clone();
80        }
81        let v = gen();
82        map.insert(key.to_string(), v.clone());
83        v
84    }
85
86    fn path_seed_lookup(comp: &str, seeds: &HashMap<String, String>) -> Option<String> {
87        if let Some(v) = seeds.get(comp) {
88            return Some(v.clone());
89        }
90        if let Some(dot) = comp.find('.') {
91            let stem = &comp[..dot];
92            if let Some(v) = seeds.get(stem) {
93                let ext = &comp[dot..];
94                return Some(format!("{v}{ext}"));
95            }
96        }
97        None
98    }
99
100    fn anonymize_path_component(&mut self, comp: &str) -> String {
101        if let Some(mapped) = Self::path_seed_lookup(comp, self.seeds) {
102            return Self::map_token(&mut self.paths, &HashMap::new(), comp, || mapped);
103        }
104        Self::map_token(&mut self.paths, self.seeds, comp, || {
105            let n = self.path_n;
106            self.path_n += 1;
107            format!("path{n}")
108        })
109    }
110
111    fn anonymize_path(&mut self, path: &str) -> String {
112        if !path.is_empty() && self.seeds.contains_key(path) {
113            return self.seeds[path].clone();
114        }
115        let mut out = String::new();
116        for (i, part) in path.split('/').enumerate() {
117            if i > 0 {
118                out.push('/');
119            }
120            out.push_str(&self.anonymize_path_component(part));
121        }
122        out
123    }
124
125    fn anonymize_refname(&mut self, refname: &str) -> String {
126        const PREFIXES: &[&str] = &["refs/heads/", "refs/tags/", "refs/remotes/", "refs/"];
127        let mut rest = refname;
128        let mut prefix = "";
129        for p in PREFIXES {
130            if let Some(stripped) = refname.strip_prefix(p) {
131                prefix = p;
132                rest = stripped;
133                break;
134            }
135        }
136        let mut out = prefix.to_string();
137        if rest.is_empty() {
138            return out;
139        }
140        for (i, comp) in rest.split('/').enumerate() {
141            if i > 0 {
142                out.push('/');
143            }
144            out.push_str(&Self::map_token(&mut self.refs, self.seeds, comp, || {
145                let n = self.ref_n;
146                self.ref_n += 1;
147                format!("ref{n}")
148            }));
149        }
150        out
151    }
152
153    fn anonymize_oid_hex(&mut self, hex: &str) -> String {
154        Self::map_token(&mut self.objs, self.seeds, hex, || {
155            self.oid_n += 1;
156            format!("{:040x}", self.oid_n as u128)
157        })
158    }
159
160    fn anonymize_ident_line(&mut self, line: &str) -> String {
161        // "author NAME <EMAIL> DATE TZ" — preserve header word and date tail.
162        let Some(space) = line.find(' ') else {
163            return line.to_owned();
164        };
165        let header = &line[..space + 1];
166        let rest = line[space + 1..].trim_end();
167        let Some(gt) = rest.rfind('>') else {
168            return format!("{header}Malformed Ident <malformed@example.com> 0 -0000");
169        };
170        let name_email = &rest[..gt + 1];
171        let after = rest[gt + 1..].trim_start();
172        let key = name_email.to_string();
173        let ident = Self::map_token(&mut self.idents, self.seeds, &key, || {
174            let n = self.ident_n;
175            self.ident_n += 1;
176            format!("User {n} <user{n}@example.com>")
177        });
178        format!("{header}{ident} {after}")
179    }
180
181    fn anonymize_commit_message(&mut self) -> String {
182        let n = self.subject_n;
183        self.subject_n += 1;
184        format!("subject {n}\n\nbody\n")
185    }
186
187    fn anonymize_tag_message(&mut self, msg: &str) -> String {
188        Self::map_token(&mut self.tag_msgs, self.seeds, msg, || {
189            let n = self.tag_msg_n;
190            self.tag_msg_n += 1;
191            format!("tag message {n}")
192        })
193    }
194
195    fn anonymize_blob_payload(&mut self) -> Vec<u8> {
196        let n = self.blob_n;
197        self.blob_n += 1;
198        format!("anonymous blob {n}").into_bytes()
199    }
200}
201
202fn parse_anonymize_maps(entries: &[String]) -> Result<HashMap<String, String>> {
203    let mut out = HashMap::new();
204    for raw in entries {
205        let raw = raw.trim();
206        if raw.is_empty() {
207            return Err(Error::InvalidRef(
208                "--anonymize-map token cannot be empty".to_owned(),
209            ));
210        }
211        if let Some((k, v)) = raw.split_once(':') {
212            if k.is_empty() || v.is_empty() {
213                return Err(Error::InvalidRef(
214                    "--anonymize-map token cannot be empty".to_owned(),
215                ));
216            }
217            out.insert(k.to_string(), v.to_string());
218        } else {
219            out.insert(raw.to_string(), raw.to_string());
220        }
221    }
222    Ok(out)
223}
224
225/// Ref tips used to assign each exported commit a `commit <ref>` line (Git `revision_sources`).
226///
227/// Includes `refs/heads/*` and peeled `refs/tags/*` so tagged-only commits (e.g. `git tag E` with no
228/// branch) still get a valid source ref. Without tags, `fast-export --all` can fail with
229/// `no ref source for commit` when the walk reaches a commit reachable only via tags.
230fn revision_source_tips(repo: &Repository) -> Result<Vec<(String, ObjectId)>> {
231    let mut tips = refs::list_refs(&repo.git_dir, "refs/heads/")?;
232    for (name, oid) in refs::list_refs(&repo.git_dir, "refs/tags/")? {
233        let tip = match peel_tag_to_commit_oid(repo, oid) {
234            Ok(c) => c,
235            Err(_) => continue,
236        };
237        tips.push((name, tip));
238    }
239    Ok(tips)
240}
241
242fn ref_source_for_commit(
243    repo: &Repository,
244    oid: ObjectId,
245    head_branches: &[(String, ObjectId)],
246) -> Result<String> {
247    let mut best: Option<(&str, usize)> = None;
248    for (name, tip) in head_branches {
249        if *tip != oid {
250            continue;
251        }
252        let score = name.len();
253        if best.is_none_or(|(_, s)| score < s) {
254            best = Some((name.as_str(), score));
255        }
256    }
257    if let Some((n, _)) = best {
258        return Ok(n.to_string());
259    }
260    // Propagate first-seen ref name along parents (matches Git `revision_sources`).
261    let mut source: HashMap<ObjectId, String> = HashMap::new();
262    let mut queue: std::collections::VecDeque<ObjectId> = std::collections::VecDeque::new();
263    for (name, tip) in head_branches {
264        if source.insert(*tip, name.clone()).is_none() {
265            queue.push_back(*tip);
266        }
267    }
268    while let Some(c) = queue.pop_front() {
269        let pname = source.get(&c).cloned().unwrap_or_default();
270        let commit = load_commit(repo, c)?;
271        for p in commit.parents {
272            if source.contains_key(&p) {
273                continue;
274            }
275            source.insert(p, pname.clone());
276            queue.push_back(p);
277        }
278    }
279    source
280        .get(&oid)
281        .cloned()
282        .ok_or_else(|| Error::InvalidRef(format!("no ref source for commit {oid}")))
283}
284
285fn load_commit(repo: &Repository, oid: ObjectId) -> Result<CommitData> {
286    let obj = repo.odb.read(&oid)?;
287    if obj.kind != ObjectKind::Commit {
288        return Err(Error::CorruptObject(format!(
289            "expected commit, got {}",
290            obj.kind.as_str()
291        )));
292    }
293    parse_commit(&obj.data)
294}
295
296fn peel_tag_to_commit_oid(repo: &Repository, mut oid: ObjectId) -> Result<ObjectId> {
297    loop {
298        let obj = repo.odb.read(&oid)?;
299        match obj.kind {
300            ObjectKind::Commit => return Ok(oid),
301            ObjectKind::Tag => {
302                let t = parse_tag(&obj.data)?;
303                oid = t.object;
304            }
305            _ => {
306                return Err(Error::CorruptObject(
307                    "tag does not point to a commit".to_owned(),
308                ));
309            }
310        }
311    }
312}
313
314fn depth_first_diff_sort(entries: &mut [DiffEntry]) {
315    entries.sort_by(|a, b| {
316        let pa = a.path();
317        let pb = b.path();
318        let la = pa.len();
319        let lb = pb.len();
320        let minlen = la.min(lb);
321        let cmp = pa.as_bytes()[..minlen].cmp(&pb.as_bytes()[..minlen]);
322        if cmp != std::cmp::Ordering::Equal {
323            return cmp;
324        }
325        let len_cmp = lb.cmp(&la);
326        if len_cmp != std::cmp::Ordering::Equal {
327            return len_cmp;
328        }
329        let ar = matches!(a.status, DiffStatus::Renamed);
330        let br = matches!(b.status, DiffStatus::Renamed);
331        ar.cmp(&br)
332    });
333}
334
335/// Write a fast-import stream for the repository to `writer`.
336///
337/// # Errors
338///
339/// Propagates object database, ref, and revision walk errors.
340pub fn export_stream(
341    repo: &Repository,
342    mut writer: impl Write,
343    options: &FastExportOptions,
344) -> Result<()> {
345    if !options.all {
346        return Err(Error::InvalidRef(
347            "fast-export: only --all is implemented".to_owned(),
348        ));
349    }
350
351    let seeds = if options.anonymize {
352        parse_anonymize_maps(&options.anonymize_maps)?
353    } else {
354        HashMap::new()
355    };
356
357    if !options.anonymize && !options.anonymize_maps.is_empty() {
358        return Err(Error::InvalidRef(
359            "the option '--anonymize-map' requires '--anonymize'".to_owned(),
360        ));
361    }
362
363    let head_branches = revision_source_tips(repo)?;
364
365    let opts = RevListOptions {
366        all_refs: true,
367        ordering: OrderingMode::Topo,
368        reverse: true,
369        ..RevListOptions::default()
370    };
371    let rev_result = rev_list(repo, &[] as &[String], &[] as &[String], &opts)?;
372    let commits: Vec<ObjectId> = rev_result.commits;
373
374    let commit_set: HashSet<ObjectId> = commits.iter().copied().collect();
375
376    let mut marks: HashMap<ObjectId, u32> = HashMap::new();
377    let mut next_mark: u32 = 0;
378
379    let mut anon = if options.anonymize {
380        Some(AnonState::new(&seeds))
381    } else {
382        None
383    };
384
385    if options.use_done_feature {
386        writeln!(writer, "feature done")?;
387    }
388
389    for oid in &commits {
390        let raw_commit = load_commit(repo, *oid)?;
391        let parent_tree = if let Some(p) = raw_commit.parents.first() {
392            let pc = load_commit(repo, *p)?;
393            Some(pc.tree)
394        } else {
395            None
396        };
397        let diffs = diff_trees(&repo.odb, parent_tree.as_ref(), Some(&raw_commit.tree), "")?;
398        let mut diff_vec: Vec<DiffEntry> = diffs
399            .into_iter()
400            .filter(|e| {
401                matches!(
402                    e.status,
403                    DiffStatus::Added
404                        | DiffStatus::Deleted
405                        | DiffStatus::Modified
406                        | DiffStatus::Renamed
407                        | DiffStatus::Copied
408                        | DiffStatus::TypeChanged
409                )
410            })
411            .collect();
412        depth_first_diff_sort(&mut diff_vec);
413
414        if !options.no_data {
415            for e in &diff_vec {
416                if e.status == DiffStatus::Deleted {
417                    continue;
418                }
419                let mode = u32::from_str_radix(e.new_mode.trim(), 8).unwrap_or(0);
420                if mode == MODE_TREE || mode == MODE_GITLINK {
421                    continue;
422                }
423                let blob_oid = e.new_oid;
424                if marks.contains_key(&blob_oid) {
425                    continue;
426                }
427                next_mark += 1;
428                marks.insert(blob_oid, next_mark);
429                writeln!(writer, "blob")?;
430                writeln!(writer, "mark :{next_mark}")?;
431                let payload = if let Some(a) = anon.as_mut() {
432                    a.anonymize_blob_payload()
433                } else {
434                    let o = repo.odb.read(&blob_oid)?;
435                    if o.kind != ObjectKind::Blob {
436                        return Err(Error::CorruptObject("expected blob".to_owned()));
437                    }
438                    o.data
439                };
440                writeln!(writer, "data {}", payload.len())?;
441                writer.write_all(&payload)?;
442                writeln!(writer)?;
443            }
444        }
445
446        let refname = ref_source_for_commit(repo, *oid, &head_branches)?;
447        let export_ref = if let Some(a) = anon.as_mut() {
448            a.anonymize_refname(&refname)
449        } else {
450            refname.clone()
451        };
452
453        if raw_commit.parents.is_empty() {
454            writeln!(writer, "reset {export_ref}")?;
455        }
456
457        next_mark += 1;
458        let commit_mark = next_mark;
459        marks.insert(*oid, commit_mark);
460
461        writeln!(writer, "commit {export_ref}")?;
462        writeln!(writer, "mark :{commit_mark}")?;
463
464        let author_line = if let Some(a) = anon.as_mut() {
465            a.anonymize_ident_line(&format!("author {}", raw_commit.author))
466        } else {
467            format!("author {}", raw_commit.author)
468        };
469        let committer_line = if let Some(a) = anon.as_mut() {
470            a.anonymize_ident_line(&format!("committer {}", raw_commit.committer))
471        } else {
472            format!("committer {}", raw_commit.committer)
473        };
474        writeln!(writer, "{author_line}")?;
475        writeln!(writer, "{committer_line}")?;
476
477        let message = if let Some(a) = anon.as_mut() {
478            a.anonymize_commit_message()
479        } else {
480            raw_commit.message.clone()
481        };
482        let msg_bytes = message.as_bytes();
483        writeln!(writer, "data {}", msg_bytes.len())?;
484        writer.write_all(msg_bytes)?;
485        writeln!(writer)?;
486
487        for (i, p) in raw_commit.parents.iter().enumerate() {
488            let label = if i == 0 { "from" } else { "merge" };
489            write!(writer, "{label} ")?;
490            if let Some(&m) = marks.get(p) {
491                writeln!(writer, ":{m}")?;
492            } else {
493                let hex = p.to_hex();
494                let out = if let Some(a) = anon.as_mut() {
495                    a.anonymize_oid_hex(&hex)
496                } else {
497                    hex
498                };
499                writeln!(writer, "{out}")?;
500            }
501        }
502
503        let mut changed: HashSet<String> = HashSet::new();
504        for e in &diff_vec {
505            match e.status {
506                DiffStatus::Deleted => {
507                    let path = if let Some(a) = anon.as_mut() {
508                        a.anonymize_path(e.path())
509                    } else {
510                        e.path().to_string()
511                    };
512                    writeln!(writer, "D {path}")?;
513                    changed.insert(e.path().to_string());
514                }
515                DiffStatus::Renamed | DiffStatus::Copied => {
516                    let old_p = e.old_path.as_deref().unwrap_or("");
517                    let skip_modify = e.old_oid == e.new_oid
518                        && e.old_mode == e.new_mode
519                        && !changed.contains(old_p);
520                    if !changed.contains(old_p) {
521                        let op = if let Some(a) = anon.as_mut() {
522                            a.anonymize_path(old_p)
523                        } else {
524                            old_p.to_string()
525                        };
526                        let np = if let Some(a) = anon.as_mut() {
527                            a.anonymize_path(e.path())
528                        } else {
529                            e.path().to_string()
530                        };
531                        writeln!(writer, "{} {op} {np}", e.status.letter())?;
532                    }
533                    if !skip_modify {
534                        fallthrough_modify(
535                            repo,
536                            &mut writer,
537                            e,
538                            &marks,
539                            anon.as_mut(),
540                            options.anonymize,
541                            options.no_data,
542                        )?;
543                    }
544                    changed.insert(old_p.to_string());
545                    changed.insert(e.path().to_string());
546                }
547                DiffStatus::Added | DiffStatus::Modified | DiffStatus::TypeChanged => {
548                    fallthrough_modify(
549                        repo,
550                        &mut writer,
551                        e,
552                        &marks,
553                        anon.as_mut(),
554                        options.anonymize,
555                        options.no_data,
556                    )?;
557                    changed.insert(e.path().to_string());
558                }
559                _ => {}
560            }
561        }
562        writeln!(writer)?;
563    }
564
565    // Annotated tags that point at exported commits
566    let tag_refs = refs::list_refs(&repo.git_dir, "refs/tags/")?;
567    for (full_name, tag_oid) in tag_refs {
568        let tag_obj = repo.odb.read(&tag_oid)?;
569        if tag_obj.kind != ObjectKind::Tag {
570            continue;
571        }
572        let tag_data = parse_tag(&tag_obj.data)?;
573        let Ok(target_commit) = peel_tag_to_commit_oid(repo, tag_data.object) else {
574            continue;
575        };
576        if !commit_set.contains(&target_commit) {
577            continue;
578        }
579        let Some(&tip_mark) = marks.get(&target_commit) else {
580            continue;
581        };
582
583        let export_name = if let Some(a) = anon.as_mut() {
584            a.anonymize_refname(&full_name)
585        } else {
586            full_name.clone()
587        };
588        let short_name = export_name
589            .strip_prefix("refs/tags/")
590            .unwrap_or(&export_name)
591            .to_string();
592
593        let tagger_line = if let Some(t) = tag_data.tagger.as_deref() {
594            if let Some(a) = anon.as_mut() {
595                a.anonymize_ident_line(&format!("tagger {t}"))
596            } else {
597                format!("tagger {t}")
598            }
599        } else {
600            String::new()
601        };
602
603        let msg = if options.anonymize {
604            anon.as_mut()
605                .map(|a| a.anonymize_tag_message(&tag_data.message))
606                .unwrap_or_default()
607        } else {
608            tag_data.message.clone()
609        };
610
611        writeln!(writer, "tag {short_name}")?;
612        writeln!(writer, "from :{tip_mark}")?;
613        if !tagger_line.is_empty() {
614            writeln!(writer, "{tagger_line}")?;
615        }
616        let msg_bytes = msg.as_bytes();
617        writeln!(writer, "data {}", msg_bytes.len())?;
618        writer.write_all(msg_bytes)?;
619        writeln!(writer)?;
620    }
621
622    if options.use_done_feature {
623        writeln!(writer, "done")?;
624    }
625
626    Ok(())
627}
628
629fn fallthrough_modify(
630    _repo: &Repository,
631    writer: &mut impl Write,
632    e: &DiffEntry,
633    marks: &HashMap<ObjectId, u32>,
634    mut anon: Option<&mut AnonState>,
635    _anonymize: bool,
636    no_data: bool,
637) -> Result<()> {
638    let mode = u32::from_str_radix(e.new_mode.trim(), 8).unwrap_or(0);
639    let path = if let Some(a) = anon.as_mut() {
640        a.anonymize_path(e.path())
641    } else {
642        e.path().to_string()
643    };
644    if mode == MODE_GITLINK {
645        let hex = e.new_oid.to_hex();
646        let oid_out = if let Some(a) = anon {
647            a.anonymize_oid_hex(&hex)
648        } else {
649            hex
650        };
651        writeln!(writer, "M {:06o} {oid_out} {path}", mode)?;
652        return Ok(());
653    }
654    if no_data {
655        let hex = e.new_oid.to_hex();
656        let oid_out = if let Some(a) = anon.as_mut() {
657            a.anonymize_oid_hex(&hex)
658        } else {
659            hex
660        };
661        writeln!(writer, "M {:06o} {oid_out} {path}", mode)?;
662        return Ok(());
663    }
664    let Some(&bm) = marks.get(&e.new_oid) else {
665        return Err(Error::IndexError(format!(
666            "fast-export: missing mark for blob {}",
667            e.new_oid
668        )));
669    };
670    writeln!(writer, "M {:06o} :{bm} {path}", mode)?;
671    Ok(())
672}
grit_lib/fast_export.rs

grit_lib/
fast_export.rs
