grit_lib/
fast_export.rs

1//! [`git fast-export`](https://git-scm.com/docs/git-fast-export) stream generation.
2//!
3//! Supports the subset needed by upstream tests: `--all`, `--anonymize`,
4//! `--anonymize-map`, topological commit order with `reverse` (oldest first),
5//! blob/commit marks, per-commit tree diffs, and annotated tags on commits.
6
7use std::collections::{HashMap, HashSet};
8use std::io::Write;
9
10use crate::diff::{diff_trees, DiffEntry, DiffStatus};
11use crate::error::{Error, Result};
12use crate::objects::{parse_commit, parse_tag, CommitData, ObjectId, ObjectKind};
13use crate::refs;
14use crate::repo::Repository;
15use crate::rev_list::{rev_list, OrderingMode, RevListOptions};
16
17use crate::index::{MODE_GITLINK, MODE_TREE};
18
19/// Options for [`export_stream`].
20#[derive(Debug, Clone, Default)]
21pub struct FastExportOptions {
22    /// Export all heads under `refs/heads/` (and reachable history).
23    pub all: bool,
24    /// Replace paths, idents, messages, and non-mark OIDs with stable placeholders.
25    pub anonymize: bool,
26    /// `from:to` or bare `token` mappings (last duplicate key wins, matching Git).
27    pub anonymize_maps: Vec<String>,
28    /// Emit `feature done` / trailing `done` (matches `git fast-import` when the feature is negotiated).
29    pub use_done_feature: bool,
30    /// Omit `blob` commands and emit `M` lines with full object ids (matches `git fast-export --no-data`).
31    pub no_data: bool,
32}
33
34struct AnonState<'a> {
35    seeds: &'a HashMap<String, String>,
36    paths: HashMap<String, String>,
37    refs: HashMap<String, String>,
38    objs: HashMap<String, String>,
39    idents: HashMap<String, String>,
40    tag_msgs: HashMap<String, String>,
41    path_n: u32,
42    ref_n: u32,
43    oid_n: u32,
44    ident_n: u32,
45    subject_n: u32,
46    tag_msg_n: u32,
47    blob_n: u32,
48}
49
50impl<'a> AnonState<'a> {
51    fn new(seeds: &'a HashMap<String, String>) -> Self {
52        Self {
53            seeds,
54            paths: HashMap::new(),
55            refs: HashMap::new(),
56            objs: HashMap::new(),
57            idents: HashMap::new(),
58            tag_msgs: HashMap::new(),
59            path_n: 0,
60            ref_n: 0,
61            oid_n: 0,
62            ident_n: 0,
63            subject_n: 0,
64            tag_msg_n: 0,
65            blob_n: 0,
66        }
67    }
68
69    fn map_token(
70        map: &mut HashMap<String, String>,
71        seeds: &HashMap<String, String>,
72        key: &str,
73        gen: impl FnOnce() -> String,
74    ) -> String {
75        if let Some(v) = seeds.get(key) {
76            return v.clone();
77        }
78        if let Some(v) = map.get(key) {
79            return v.clone();
80        }
81        let v = gen();
82        map.insert(key.to_string(), v.clone());
83        v
84    }
85
86    fn path_seed_lookup(comp: &str, seeds: &HashMap<String, String>) -> Option<String> {
87        if let Some(v) = seeds.get(comp) {
88            return Some(v.clone());
89        }
90        if let Some(dot) = comp.find('.') {
91            let stem = &comp[..dot];
92            if let Some(v) = seeds.get(stem) {
93                let ext = &comp[dot..];
94                return Some(format!("{v}{ext}"));
95            }
96        }
97        None
98    }
99
100    fn anonymize_path_component(&mut self, comp: &str) -> String {
101        if let Some(mapped) = Self::path_seed_lookup(comp, self.seeds) {
102            return Self::map_token(&mut self.paths, &HashMap::new(), comp, || mapped);
103        }
104        Self::map_token(&mut self.paths, self.seeds, comp, || {
105            let n = self.path_n;
106            self.path_n += 1;
107            format!("path{n}")
108        })
109    }
110
111    fn anonymize_path(&mut self, path: &str) -> String {
112        if !path.is_empty() && self.seeds.contains_key(path) {
113            return self.seeds[path].clone();
114        }
115        let mut out = String::new();
116        for (i, part) in path.split('/').enumerate() {
117            if i > 0 {
118                out.push('/');
119            }
120            out.push_str(&self.anonymize_path_component(part));
121        }
122        out
123    }
124
125    fn anonymize_refname(&mut self, refname: &str) -> String {
126        const PREFIXES: &[&str] = &["refs/heads/", "refs/tags/", "refs/remotes/", "refs/"];
127        let mut rest = refname;
128        let mut prefix = "";
129        for p in PREFIXES {
130            if let Some(stripped) = refname.strip_prefix(p) {
131                prefix = p;
132                rest = stripped;
133                break;
134            }
135        }
136        let mut out = prefix.to_string();
137        if rest.is_empty() {
138            return out;
139        }
140        for (i, comp) in rest.split('/').enumerate() {
141            if i > 0 {
142                out.push('/');
143            }
144            out.push_str(&Self::map_token(&mut self.refs, self.seeds, comp, || {
145                let n = self.ref_n;
146                self.ref_n += 1;
147                format!("ref{n}")
148            }));
149        }
150        out
151    }
152
153    fn anonymize_oid_hex(&mut self, hex: &str) -> String {
154        Self::map_token(&mut self.objs, self.seeds, hex, || {
155            self.oid_n += 1;
156            format!("{:040x}", self.oid_n as u128)
157        })
158    }
159
160    fn anonymize_ident_line(&mut self, line: &str) -> String {
161        // "author NAME <EMAIL> DATE TZ" — preserve header word and date tail.
162        let Some(space) = line.find(' ') else {
163            return line.to_owned();
164        };
165        let header = &line[..space + 1];
166        let rest = line[space + 1..].trim_end();
167        let Some(gt) = rest.rfind('>') else {
168            return format!("{header}Malformed Ident <malformed@example.com> 0 -0000");
169        };
170        let name_email = &rest[..gt + 1];
171        let after = rest[gt + 1..].trim_start();
172        let key = name_email.to_string();
173        let ident = Self::map_token(&mut self.idents, self.seeds, &key, || {
174            let n = self.ident_n;
175            self.ident_n += 1;
176            format!("User {n} <user{n}@example.com>")
177        });
178        format!("{header}{ident} {after}")
179    }
180
181    fn anonymize_commit_message(&mut self) -> String {
182        let n = self.subject_n;
183        self.subject_n += 1;
184        format!("subject {n}\n\nbody\n")
185    }
186
187    fn anonymize_tag_message(&mut self, msg: &str) -> String {
188        Self::map_token(&mut self.tag_msgs, self.seeds, msg, || {
189            let n = self.tag_msg_n;
190            self.tag_msg_n += 1;
191            format!("tag message {n}")
192        })
193    }
194
195    fn anonymize_blob_payload(&mut self) -> Vec<u8> {
196        let n = self.blob_n;
197        self.blob_n += 1;
198        format!("anonymous blob {n}").into_bytes()
199    }
200}
201
202fn parse_anonymize_maps(entries: &[String]) -> Result<HashMap<String, String>> {
203    let mut out = HashMap::new();
204    for raw in entries {
205        let raw = raw.trim();
206        if raw.is_empty() {
207            return Err(Error::InvalidRef(
208                "--anonymize-map token cannot be empty".to_owned(),
209            ));
210        }
211        if let Some((k, v)) = raw.split_once(':') {
212            if k.is_empty() || v.is_empty() {
213                return Err(Error::InvalidRef(
214                    "--anonymize-map token cannot be empty".to_owned(),
215                ));
216            }
217            out.insert(k.to_string(), v.to_string());
218        } else {
219            out.insert(raw.to_string(), raw.to_string());
220        }
221    }
222    Ok(out)
223}
224
225fn ref_source_for_commit(
226    repo: &Repository,
227    oid: ObjectId,
228    head_branches: &[(String, ObjectId)],
229) -> Result<String> {
230    let mut best: Option<(&str, usize)> = None;
231    for (name, tip) in head_branches {
232        if *tip != oid {
233            continue;
234        }
235        let score = name.len();
236        if best.is_none_or(|(_, s)| score < s) {
237            best = Some((name.as_str(), score));
238        }
239    }
240    if let Some((n, _)) = best {
241        return Ok(n.to_string());
242    }
243    // Propagate first-seen ref name along parents (matches Git `revision_sources`).
244    let mut source: HashMap<ObjectId, String> = HashMap::new();
245    let mut queue: std::collections::VecDeque<ObjectId> = std::collections::VecDeque::new();
246    for (name, tip) in head_branches {
247        if source.insert(*tip, name.clone()).is_none() {
248            queue.push_back(*tip);
249        }
250    }
251    while let Some(c) = queue.pop_front() {
252        let pname = source.get(&c).cloned().unwrap_or_default();
253        let commit = load_commit(repo, c)?;
254        for p in commit.parents {
255            if source.contains_key(&p) {
256                continue;
257            }
258            source.insert(p, pname.clone());
259            queue.push_back(p);
260        }
261    }
262    source
263        .get(&oid)
264        .cloned()
265        .ok_or_else(|| Error::InvalidRef(format!("no ref source for commit {oid}")))
266}
267
268fn load_commit(repo: &Repository, oid: ObjectId) -> Result<CommitData> {
269    let obj = repo.odb.read(&oid)?;
270    if obj.kind != ObjectKind::Commit {
271        return Err(Error::CorruptObject(format!(
272            "expected commit, got {}",
273            obj.kind.as_str()
274        )));
275    }
276    parse_commit(&obj.data)
277}
278
279fn peel_tag_to_commit_oid(repo: &Repository, mut oid: ObjectId) -> Result<ObjectId> {
280    loop {
281        let obj = repo.odb.read(&oid)?;
282        match obj.kind {
283            ObjectKind::Commit => return Ok(oid),
284            ObjectKind::Tag => {
285                let t = parse_tag(&obj.data)?;
286                oid = t.object;
287            }
288            _ => {
289                return Err(Error::CorruptObject(
290                    "tag does not point to a commit".to_owned(),
291                ));
292            }
293        }
294    }
295}
296
297fn depth_first_diff_sort(entries: &mut [DiffEntry]) {
298    entries.sort_by(|a, b| {
299        let pa = a.path();
300        let pb = b.path();
301        let la = pa.len();
302        let lb = pb.len();
303        let minlen = la.min(lb);
304        let cmp = pa.as_bytes()[..minlen].cmp(&pb.as_bytes()[..minlen]);
305        if cmp != std::cmp::Ordering::Equal {
306            return cmp;
307        }
308        let len_cmp = lb.cmp(&la);
309        if len_cmp != std::cmp::Ordering::Equal {
310            return len_cmp;
311        }
312        let ar = matches!(a.status, DiffStatus::Renamed);
313        let br = matches!(b.status, DiffStatus::Renamed);
314        ar.cmp(&br)
315    });
316}
317
318/// Write a fast-import stream for the repository to `writer`.
319///
320/// # Errors
321///
322/// Propagates object database, ref, and revision walk errors.
323pub fn export_stream(
324    repo: &Repository,
325    mut writer: impl Write,
326    options: &FastExportOptions,
327) -> Result<()> {
328    if !options.all {
329        return Err(Error::InvalidRef(
330            "fast-export: only --all is implemented".to_owned(),
331        ));
332    }
333
334    let seeds = if options.anonymize {
335        parse_anonymize_maps(&options.anonymize_maps)?
336    } else {
337        HashMap::new()
338    };
339
340    if !options.anonymize && !options.anonymize_maps.is_empty() {
341        return Err(Error::InvalidRef(
342            "the option '--anonymize-map' requires '--anonymize'".to_owned(),
343        ));
344    }
345
346    let head_branches: Vec<(String, ObjectId)> = refs::list_refs(&repo.git_dir, "refs/heads/")?;
347
348    let opts = RevListOptions {
349        all_refs: true,
350        ordering: OrderingMode::Topo,
351        reverse: true,
352        ..RevListOptions::default()
353    };
354    let rev_result = rev_list(repo, &[] as &[String], &[] as &[String], &opts)?;
355    let commits: Vec<ObjectId> = rev_result.commits;
356
357    let commit_set: HashSet<ObjectId> = commits.iter().copied().collect();
358
359    let mut marks: HashMap<ObjectId, u32> = HashMap::new();
360    let mut next_mark: u32 = 0;
361
362    let mut anon = if options.anonymize {
363        Some(AnonState::new(&seeds))
364    } else {
365        None
366    };
367
368    if options.use_done_feature {
369        writeln!(writer, "feature done")?;
370    }
371
372    for oid in &commits {
373        let raw_commit = load_commit(repo, *oid)?;
374        let parent_tree = if let Some(p) = raw_commit.parents.first() {
375            let pc = load_commit(repo, *p)?;
376            Some(pc.tree)
377        } else {
378            None
379        };
380        let diffs = diff_trees(&repo.odb, parent_tree.as_ref(), Some(&raw_commit.tree), "")?;
381        let mut diff_vec: Vec<DiffEntry> = diffs
382            .into_iter()
383            .filter(|e| {
384                matches!(
385                    e.status,
386                    DiffStatus::Added
387                        | DiffStatus::Deleted
388                        | DiffStatus::Modified
389                        | DiffStatus::Renamed
390                        | DiffStatus::Copied
391                        | DiffStatus::TypeChanged
392                )
393            })
394            .collect();
395        depth_first_diff_sort(&mut diff_vec);
396
397        if !options.no_data {
398            for e in &diff_vec {
399                if e.status == DiffStatus::Deleted {
400                    continue;
401                }
402                let mode = u32::from_str_radix(e.new_mode.trim(), 8).unwrap_or(0);
403                if mode == MODE_TREE || mode == MODE_GITLINK {
404                    continue;
405                }
406                let blob_oid = e.new_oid;
407                if marks.contains_key(&blob_oid) {
408                    continue;
409                }
410                next_mark += 1;
411                marks.insert(blob_oid, next_mark);
412                writeln!(writer, "blob")?;
413                writeln!(writer, "mark :{next_mark}")?;
414                let payload = if let Some(a) = anon.as_mut() {
415                    a.anonymize_blob_payload()
416                } else {
417                    let o = repo.odb.read(&blob_oid)?;
418                    if o.kind != ObjectKind::Blob {
419                        return Err(Error::CorruptObject("expected blob".to_owned()));
420                    }
421                    o.data
422                };
423                writeln!(writer, "data {}", payload.len())?;
424                writer.write_all(&payload)?;
425                writeln!(writer)?;
426            }
427        }
428
429        let refname = ref_source_for_commit(repo, *oid, &head_branches)?;
430        let export_ref = if let Some(a) = anon.as_mut() {
431            a.anonymize_refname(&refname)
432        } else {
433            refname.clone()
434        };
435
436        if raw_commit.parents.is_empty() {
437            writeln!(writer, "reset {export_ref}")?;
438        }
439
440        next_mark += 1;
441        let commit_mark = next_mark;
442        marks.insert(*oid, commit_mark);
443
444        writeln!(writer, "commit {export_ref}")?;
445        writeln!(writer, "mark :{commit_mark}")?;
446
447        let author_line = if let Some(a) = anon.as_mut() {
448            a.anonymize_ident_line(&format!("author {}", raw_commit.author))
449        } else {
450            format!("author {}", raw_commit.author)
451        };
452        let committer_line = if let Some(a) = anon.as_mut() {
453            a.anonymize_ident_line(&format!("committer {}", raw_commit.committer))
454        } else {
455            format!("committer {}", raw_commit.committer)
456        };
457        writeln!(writer, "{author_line}")?;
458        writeln!(writer, "{committer_line}")?;
459
460        let message = if let Some(a) = anon.as_mut() {
461            a.anonymize_commit_message()
462        } else {
463            raw_commit.message.clone()
464        };
465        let msg_bytes = message.as_bytes();
466        writeln!(writer, "data {}", msg_bytes.len())?;
467        writer.write_all(msg_bytes)?;
468        writeln!(writer)?;
469
470        for (i, p) in raw_commit.parents.iter().enumerate() {
471            let label = if i == 0 { "from" } else { "merge" };
472            write!(writer, "{label} ")?;
473            if let Some(&m) = marks.get(p) {
474                writeln!(writer, ":{m}")?;
475            } else {
476                let hex = p.to_hex();
477                let out = if let Some(a) = anon.as_mut() {
478                    a.anonymize_oid_hex(&hex)
479                } else {
480                    hex
481                };
482                writeln!(writer, "{out}")?;
483            }
484        }
485
486        let mut changed: HashSet<String> = HashSet::new();
487        for e in &diff_vec {
488            match e.status {
489                DiffStatus::Deleted => {
490                    let path = if let Some(a) = anon.as_mut() {
491                        a.anonymize_path(e.path())
492                    } else {
493                        e.path().to_string()
494                    };
495                    writeln!(writer, "D {path}")?;
496                    changed.insert(e.path().to_string());
497                }
498                DiffStatus::Renamed | DiffStatus::Copied => {
499                    let old_p = e.old_path.as_deref().unwrap_or("");
500                    let skip_modify = e.old_oid == e.new_oid
501                        && e.old_mode == e.new_mode
502                        && !changed.contains(old_p);
503                    if !changed.contains(old_p) {
504                        let op = if let Some(a) = anon.as_mut() {
505                            a.anonymize_path(old_p)
506                        } else {
507                            old_p.to_string()
508                        };
509                        let np = if let Some(a) = anon.as_mut() {
510                            a.anonymize_path(e.path())
511                        } else {
512                            e.path().to_string()
513                        };
514                        writeln!(writer, "{} {op} {np}", e.status.letter())?;
515                    }
516                    if !skip_modify {
517                        fallthrough_modify(
518                            repo,
519                            &mut writer,
520                            e,
521                            &marks,
522                            anon.as_mut(),
523                            options.anonymize,
524                            options.no_data,
525                        )?;
526                    }
527                    changed.insert(old_p.to_string());
528                    changed.insert(e.path().to_string());
529                }
530                DiffStatus::Added | DiffStatus::Modified | DiffStatus::TypeChanged => {
531                    fallthrough_modify(
532                        repo,
533                        &mut writer,
534                        e,
535                        &marks,
536                        anon.as_mut(),
537                        options.anonymize,
538                        options.no_data,
539                    )?;
540                    changed.insert(e.path().to_string());
541                }
542                _ => {}
543            }
544        }
545        writeln!(writer)?;
546    }
547
548    // Annotated tags that point at exported commits
549    let tag_refs = refs::list_refs(&repo.git_dir, "refs/tags/")?;
550    for (full_name, tag_oid) in tag_refs {
551        let tag_obj = repo.odb.read(&tag_oid)?;
552        if tag_obj.kind != ObjectKind::Tag {
553            continue;
554        }
555        let tag_data = parse_tag(&tag_obj.data)?;
556        let Ok(target_commit) = peel_tag_to_commit_oid(repo, tag_data.object) else {
557            continue;
558        };
559        if !commit_set.contains(&target_commit) {
560            continue;
561        }
562        let Some(&tip_mark) = marks.get(&target_commit) else {
563            continue;
564        };
565
566        let export_name = if let Some(a) = anon.as_mut() {
567            a.anonymize_refname(&full_name)
568        } else {
569            full_name.clone()
570        };
571        let short_name = export_name
572            .strip_prefix("refs/tags/")
573            .unwrap_or(&export_name)
574            .to_string();
575
576        let tagger_line = if let Some(t) = tag_data.tagger.as_deref() {
577            if let Some(a) = anon.as_mut() {
578                a.anonymize_ident_line(&format!("tagger {t}"))
579            } else {
580                format!("tagger {t}")
581            }
582        } else {
583            String::new()
584        };
585
586        let msg = if options.anonymize {
587            anon.as_mut()
588                .map(|a| a.anonymize_tag_message(&tag_data.message))
589                .unwrap_or_default()
590        } else {
591            tag_data.message.clone()
592        };
593
594        writeln!(writer, "tag {short_name}")?;
595        writeln!(writer, "from :{tip_mark}")?;
596        if !tagger_line.is_empty() {
597            writeln!(writer, "{tagger_line}")?;
598        }
599        let msg_bytes = msg.as_bytes();
600        writeln!(writer, "data {}", msg_bytes.len())?;
601        writer.write_all(msg_bytes)?;
602        writeln!(writer)?;
603    }
604
605    if options.use_done_feature {
606        writeln!(writer, "done")?;
607    }
608
609    Ok(())
610}
611
612fn fallthrough_modify(
613    _repo: &Repository,
614    writer: &mut impl Write,
615    e: &DiffEntry,
616    marks: &HashMap<ObjectId, u32>,
617    mut anon: Option<&mut AnonState>,
618    _anonymize: bool,
619    no_data: bool,
620) -> Result<()> {
621    let mode = u32::from_str_radix(e.new_mode.trim(), 8).unwrap_or(0);
622    let path = if let Some(a) = anon.as_mut() {
623        a.anonymize_path(e.path())
624    } else {
625        e.path().to_string()
626    };
627    if mode == MODE_GITLINK {
628        let hex = e.new_oid.to_hex();
629        let oid_out = if let Some(a) = anon {
630            a.anonymize_oid_hex(&hex)
631        } else {
632            hex
633        };
634        writeln!(writer, "M {:06o} {oid_out} {path}", mode)?;
635        return Ok(());
636    }
637    if no_data {
638        let hex = e.new_oid.to_hex();
639        let oid_out = if let Some(a) = anon.as_mut() {
640            a.anonymize_oid_hex(&hex)
641        } else {
642            hex
643        };
644        writeln!(writer, "M {:06o} {oid_out} {path}", mode)?;
645        return Ok(());
646    }
647    let Some(&bm) = marks.get(&e.new_oid) else {
648        return Err(Error::IndexError(format!(
649            "fast-export: missing mark for blob {}",
650            e.new_oid
651        )));
652    };
653    writeln!(writer, "M {:06o} :{bm} {path}", mode)?;
654    Ok(())
655}
grit_lib/fast_export.rs

grit_lib/
fast_export.rs
