fluidattacks_core_git/clone/
mod.rs

1pub mod codecommit;
2pub mod https;
3pub mod ssh;
4
5pub(crate) mod reason;
6
7pub use https::build_https_config_args;
8pub use https::format_https_url;
9pub use reason::hide;
10pub use ssh::setup_ssh_env;
11
12use anyhow::{bail, Context, Result};
13use serde_json::json;
14use std::ffi::OsString;
15use std::fs;
16use std::path::{Path, PathBuf};
17use tokio::process::Command;
18use tracing::{info, warn};
19
20use crate::errors::{CloneError, FileError};
21use crate::types::{CredentialKind, Credentials, GitRoot};
22use https::HttpsAccess;
23
24/// Which refs a clone takes, and what it leaves on disk.
25#[derive(Clone, Copy, Debug, PartialEq, Eq)]
26pub enum CloneShape {
27    /// The named branch and only it, with a working tree.
28    SingleBranch,
29    /// Every ref the remote advertises, as a bare mirror.
30    Mirror,
31    /// Every branch head — `refs/heads/*` plus `HEAD` — as a bare repo:
32    /// no tags, no hidden refs, no mirror config remnants.
33    BareHeads,
34}
35
36pub struct CloneOpts {
37    pub group_name: String,
38    pub nickname: String,
39    pub repo_url: String,
40    pub branch: String,
41    pub credentials: Option<Credentials>,
42    pub follow_redirects: bool,
43    pub skip_existing: bool,
44    pub shape: CloneShape,
45}
46
47impl CloneOpts {
48    #[must_use]
49    pub fn from_root(
50        group: &str,
51        root: &GitRoot,
52        credentials: Option<Credentials>,
53        skip_existing: bool,
54        shape: CloneShape,
55    ) -> Self {
56        Self {
57            group_name: group.to_owned(),
58            nickname: root.nickname.clone(),
59            repo_url: root.url.clone(),
60            branch: root.branch.clone(),
61            credentials,
62            follow_redirects: root.follow_redirects(),
63            skip_existing,
64            shape,
65        }
66    }
67}
68
69async fn dispatch_clone(opts: &CloneOpts, dest: &Path) -> Result<()> {
70    match &opts.credentials {
71        Some(Credentials {
72            kind: CredentialKind::Ssh { .. },
73        }) => ssh::clone_ssh(opts, dest).await,
74        Some(Credentials {
75            kind: CredentialKind::Https { .. },
76        }) => https::clone_https(opts, dest).await,
77        Some(Credentials {
78            kind: CredentialKind::Token { .. },
79        }) => https::clone_https_token(opts, dest).await,
80        Some(Credentials {
81            kind: CredentialKind::AwsRole { .. },
82        }) => codecommit::clone_codecommit(opts, dest).await,
83        None if opts.repo_url.starts_with("http") => https::clone_https_public(opts, dest).await,
84        _ => bail!("no suitable credentials found for root {:?}", opts.nickname),
85    }
86}
87
88pub async fn clone_root(opts: &CloneOpts) -> Result<()> {
89    let repo_dir = PathBuf::from("groups")
90        .join(&opts.group_name)
91        .join(&opts.nickname);
92
93    if opts.skip_existing {
94        if let Ok(entries) = fs::read_dir(&repo_dir) {
95            if entries.count() > 0 {
96                info!(nickname = opts.nickname, "skipping existing repo");
97                return Ok(());
98            }
99        }
100    }
101
102    // Use tempdir_in targeting repo_dir's parent to avoid cross-filesystem rename (EXDEV)
103    let tmp_parent = repo_dir.parent().unwrap_or_else(|| Path::new("."));
104    fs::create_dir_all(tmp_parent)
105        .with_context(|| format!("creating parent dir {}", tmp_parent.display()))?;
106    let tmp_dir = tempfile::tempdir_in(tmp_parent).context("creating temp dir")?;
107    let clone_dest = tmp_dir.path().join(&opts.nickname);
108
109    dispatch_clone(opts, &clone_dest)
110        .await
111        .with_context(|| format!("cloning {:?}", opts.nickname))?;
112
113    if opts.shape == CloneShape::Mirror {
114        convert_bare_to_worktree(&clone_dest)
115            .await
116            .with_context(|| format!("converting mirror to worktree for {:?}", opts.nickname))?;
117    }
118
119    if repo_dir.exists() {
120        fs::remove_dir_all(&repo_dir)
121            .with_context(|| format!("removing old repo dir {}", repo_dir.display()))?;
122    }
123    fs::rename(&clone_dest, &repo_dir)
124        .with_context(|| format!("moving cloned repo to {}", repo_dir.display()))?;
125
126    log_clone_result(&repo_dir, opts).await;
127
128    if opts.shape == CloneShape::Mirror {
129        if let Err(e) = write_mirror_info(&repo_dir, opts) {
130            warn!(error = %e, "failed to write .info.json");
131        }
132    }
133
134    Ok(())
135}
136
137async fn convert_bare_to_worktree(clone_dest: &Path) -> Result<()> {
138    let bare_tmp = clone_dest.with_extension("bare");
139    fs::rename(clone_dest, &bare_tmp)
140        .with_context(|| format!("renaming bare repo to {}", bare_tmp.display()))?;
141    fs::create_dir_all(clone_dest)
142        .with_context(|| format!("creating worktree dir {}", clone_dest.display()))?;
143    let git_dir = clone_dest.join(".git");
144    fs::rename(&bare_tmp, &git_dir)
145        .with_context(|| format!("moving bare contents to {}", git_dir.display()))?;
146
147    let dest_str = clone_dest.to_string_lossy().to_string();
148
149    let config_out = Command::new("git")
150        .args(["-C", &dest_str, "config", "core.bare", "false"])
151        .output()
152        .await
153        .context("running git config core.bare false")?;
154    if !config_out.status.success() {
155        bail!(
156            "git config core.bare false failed: {}",
157            String::from_utf8_lossy(&config_out.stderr)
158        );
159    }
160
161    let reset_out = Command::new("git")
162        .args(["-C", &dest_str, "reset", "--hard", "HEAD"])
163        .output()
164        .await
165        .context("running git reset --hard HEAD")?;
166    if !reset_out.status.success() {
167        bail!(
168            "git reset --hard HEAD failed: {}",
169            String::from_utf8_lossy(&reset_out.stderr)
170        );
171    }
172
173    Ok(())
174}
175
176async fn log_clone_result(repo_dir: &Path, opts: &CloneOpts) {
177    let label = match opts.shape {
178        CloneShape::Mirror => "cloned (mirror)",
179        CloneShape::BareHeads => "cloned (bare heads)",
180        CloneShape::SingleBranch => "cloned",
181    };
182    let repo_dir_str = repo_dir.to_string_lossy().to_string();
183    match Command::new("git")
184        .args(["-C", &repo_dir_str, "log", "-1", "--format=%H %ai"])
185        .output()
186        .await
187    {
188        Ok(output) if output.status.success() => {
189            let head = String::from_utf8_lossy(&output.stdout).trim().to_owned();
190            info!(nickname = opts.nickname, head = head, label);
191        }
192        _ => {
193            info!(nickname = opts.nickname, dir = %repo_dir.display(), label);
194        }
195    }
196}
197
198fn write_mirror_info(repo_dir: &Path, opts: &CloneOpts) -> Result<(), FileError> {
199    let info = json!({
200        "fluid_branch": opts.branch,
201        "repo": opts.repo_url,
202    });
203    let path = repo_dir.join(".info.json");
204    fs::write(&path, format!("{info:#}")).map_err(|source| FileError {
205        action: "writing",
206        path,
207        source,
208    })
209}
210
211/// git creates whatever part of `temp_dir` is missing.
212#[expect(
213    clippy::too_many_arguments,
214    reason = "what the python https_clone, ssh_clone and call_codecommit_clone take, the credential in one"
215)]
216pub async fn clone_to_temp(
217    temp_dir: &Path,
218    url: &str,
219    branch: &str,
220    credential: Option<&CredentialKind>,
221    follow_redirects: bool,
222    mirror: bool,
223) -> Result<PathBuf, CloneError> {
224    let nickname = uuid::Uuid::new_v4().to_string();
225    let dest = temp_dir.join(&nickname);
226    let opts = CloneOpts {
227        group_name: String::new(),
228        nickname,
229        repo_url: url.to_owned(),
230        branch: branch.to_owned(),
231        credentials: None,
232        follow_redirects,
233        skip_existing: false,
234        shape: if mirror {
235            CloneShape::Mirror
236        } else {
237            CloneShape::SingleBranch
238        },
239    };
240    let cloned = match credential {
241        Some(CredentialKind::Ssh { key }) => ssh::run_git_clone(&opts, &dest, key).await,
242        Some(CredentialKind::AwsRole {
243            arn,
244            external_id,
245            via,
246        }) => codecommit::run_git_clone(&opts, &dest, arn, external_id, via.as_deref()).await,
247        _ => {
248            let access = HttpsAccess::new(url, credential)?;
249            Ok(https::run_git_clone(&opts, &dest, &access).await?)
250        }
251    };
252    cloned.inspect_err(|_| {
253        let _ = fs::remove_dir_all(&dest);
254    })?;
255    log_clone_result(&dest, &opts).await;
256    if mirror {
257        keep_with_info(&dest, &opts)?;
258    }
259    Ok(dest)
260}
261
262fn keep_with_info(dest: &Path, opts: &CloneOpts) -> Result<(), CloneError> {
263    // On an error the caller never gets the path, so nobody would remove this clone.
264    write_mirror_info(dest, opts).map_err(|error| {
265        let _ = fs::remove_dir_all(dest);
266        error.into()
267    })
268}
269
270#[must_use]
271pub fn build_clone_args(
272    opts: &CloneOpts,
273    repo_url: &str,
274    dest: &Path,
275    extra_config: &[String],
276) -> Vec<OsString> {
277    let mut args = vec!["-c".to_owned(), "core.symlinks=false".to_owned()];
278
279    args.extend(extra_config.iter().cloned());
280
281    args.push("clone".to_owned());
282
283    match opts.shape {
284        CloneShape::Mirror => args.push("--mirror".to_owned()),
285        // Every branch head and nothing else: unlike --mirror this skips
286        // refs/pull, refs/merge-requests and keep-around refs, and writes no
287        // mirror refspec into the clone's config.
288        CloneShape::BareHeads => args.extend(["--bare".to_owned(), "--no-tags".to_owned()]),
289        // An empty branch is passed on for git to refuse, not read as the default one.
290        CloneShape::SingleBranch => args.extend([
291            "--branch".to_owned(),
292            opts.branch.clone(),
293            "--single-branch".to_owned(),
294        ]),
295    }
296
297    args.extend(["--".to_owned(), repo_url.to_owned()]);
298
299    let mut args: Vec<OsString> = args.into_iter().map(OsString::from).collect();
300    // A path need not be UTF-8, so git gets its bytes.
301    args.push(dest.as_os_str().to_owned());
302    args
303}
304
305pub(crate) fn redirect_config(follow_redirects: bool) -> [String; 2] {
306    [
307        "-c".to_owned(),
308        format!("http.followRedirects={follow_redirects}"),
309    ]
310}
311
312#[cfg(test)]
313mod tests {
314    use std::ffi::{OsStr, OsString};
315    use std::os::unix::ffi::OsStrExt;
316    use std::path::Path;
317    use std::process::Command;
318
319    use super::{build_clone_args, clone_to_temp, keep_with_info, CloneOpts, CloneShape};
320    use crate::errors::CloneError;
321
322    pub(super) fn opts(shape: CloneShape) -> CloneOpts {
323        CloneOpts {
324            group_name: "a-group".to_owned(),
325            nickname: "a_root".to_owned(),
326            repo_url: "https://gitlab.com/org/repo".to_owned(),
327            branch: "trunk".to_owned(),
328            credentials: None,
329            follow_redirects: false,
330            skip_existing: false,
331            shape,
332        }
333    }
334
335    fn shape_args(shape: CloneShape) -> Vec<OsString> {
336        let opts = opts(shape);
337
338        // Between "clone" and "--" sits exactly the shape.
339        build_clone_args(&opts, &opts.repo_url, Path::new("/tmp/a_root"), &[])
340            .iter()
341            .skip_while(|arg| *arg != "clone")
342            .skip(1)
343            .take_while(|arg| *arg != "--")
344            .cloned()
345            .collect()
346    }
347
348    fn run_git(dir: &Path, args: &[&str]) -> String {
349        let output = Command::new("git")
350            .arg("-C")
351            .arg(dir)
352            .args([
353                "-c",
354                "user.email=test@example.com",
355                "-c",
356                "user.name=Test",
357                "-c",
358                "commit.gpgsign=false",
359                "-c",
360                "tag.gpgsign=false",
361            ])
362            .args(args)
363            .output()
364            .expect("git runs");
365        assert!(
366            output.status.success(),
367            "git {args:?} failed: {}",
368            String::from_utf8_lossy(&output.stderr)
369        );
370        String::from_utf8_lossy(&output.stdout).trim().to_owned()
371    }
372
373    #[test]
374    fn a_clone_never_writes_a_symbolic_link() {
375        let opts = opts(CloneShape::SingleBranch);
376        let args = build_clone_args(&opts, &opts.repo_url, Path::new("/tmp/a_root"), &[]);
377
378        assert!(args
379            .windows(2)
380            .any(|pair| pair == ["-c", "core.symlinks=false"]));
381    }
382
383    #[test]
384    fn a_destination_that_is_not_utf8_reaches_git_as_its_bytes() {
385        let opts = opts(CloneShape::SingleBranch);
386        let dest = Path::new(OsStr::from_bytes(b"/tmp/caf\xe9"));
387
388        let args = build_clone_args(&opts, &opts.repo_url, dest, &[]);
389
390        assert_eq!(args.last().map(OsString::as_os_str), Some(dest.as_os_str()));
391    }
392
393    #[test]
394    fn a_branch_clone_is_single_branch() {
395        assert_eq!(
396            shape_args(CloneShape::SingleBranch),
397            ["--branch", "trunk", "--single-branch"]
398        );
399    }
400
401    #[test]
402    fn an_empty_branch_is_asked_for_instead_of_the_default_one() {
403        let opts = CloneOpts {
404            branch: String::new(),
405            ..opts(CloneShape::SingleBranch)
406        };
407        let args = build_clone_args(&opts, &opts.repo_url, Path::new("/tmp/a_root"), &[]);
408
409        assert!(args.windows(2).any(|pair| pair == ["--branch", ""]));
410    }
411
412    #[test]
413    fn a_mirror_clone_takes_every_ref() {
414        assert_eq!(shape_args(CloneShape::Mirror), ["--mirror"]);
415    }
416
417    #[test]
418    fn a_bare_heads_clone_takes_heads_without_tags() {
419        assert_eq!(shape_args(CloneShape::BareHeads), ["--bare", "--no-tags"]);
420    }
421
422    #[test]
423    fn a_bare_heads_clone_holds_exactly_the_branch_heads() {
424        let tmp = tempfile::tempdir().expect("tempdir");
425        let source = tmp.path().join("source");
426        std::fs::create_dir(&source).expect("mkdir");
427        run_git(&source, &["init", "--initial-branch=main"]);
428        std::fs::write(source.join("file.txt"), "content\n").expect("write");
429        run_git(&source, &["add", "file.txt"]);
430        run_git(&source, &["commit", "-m", "first"]);
431        run_git(&source, &["branch", "feature"]);
432        run_git(&source, &["tag", "v1"]);
433        // The kind of hidden ref --mirror would drag in.
434        run_git(&source, &["update-ref", "refs/pull/1/head", "HEAD"]);
435
436        let dest = tmp.path().join("dest");
437        let opts = opts(CloneShape::BareHeads);
438        let args = build_clone_args(&opts, &source.to_string_lossy(), &dest, &[]);
439        let clone = Command::new("git").args(&args).output().expect("git runs");
440        assert!(
441            clone.status.success(),
442            "clone failed: {}",
443            String::from_utf8_lossy(&clone.stderr)
444        );
445
446        let refs = run_git(&dest, &["for-each-ref", "--format=%(refname)"]);
447        let mut held: Vec<&str> = refs.lines().collect();
448        held.sort_unstable();
449        assert_eq!(held, ["refs/heads/feature", "refs/heads/main"]);
450        assert_eq!(run_git(&dest, &["symbolic-ref", "HEAD"]), "refs/heads/main");
451        assert_eq!(run_git(&dest, &["config", "core.bare"]), "true");
452        // No mirror remnants: neither the flag nor a fetch refspec.
453        for key in ["remote.origin.mirror", "remote.origin.fetch"] {
454            let lookup = Command::new("git")
455                .args(["-C", &dest.to_string_lossy(), "config", "--get", key])
456                .output()
457                .expect("git runs");
458            assert!(!lookup.status.success(), "{key} is set");
459        }
460    }
461
462    pub(super) fn source(tmp: &Path) -> String {
463        let work = tmp.join("work");
464        std::fs::create_dir(&work).unwrap();
465        run_git(&work, &["init", "--quiet", "--initial-branch=main"]);
466        run_git(&work, &["commit", "--quiet", "--allow-empty", "-m", "one"]);
467        format!("file://{}", work.display())
468    }
469
470    #[tokio::test]
471    async fn git_creates_a_missing_temp_dir() {
472        let tmp = tempfile::tempdir().unwrap();
473        let url = source(tmp.path());
474        let temp_dir = tmp.path().join("missing/clones");
475
476        let folder = clone_to_temp(&temp_dir, &url, "main", None, false, false)
477            .await
478            .unwrap();
479
480        assert_eq!(folder.parent(), Some(temp_dir.as_path()));
481    }
482
483    #[tokio::test]
484    async fn a_url_without_a_credential_goes_to_git_as_given() {
485        let tmp = tempfile::tempdir().unwrap();
486        let url = source(tmp.path());
487
488        let folder = clone_to_temp(tmp.path(), &url, "main", None, false, false)
489            .await
490            .unwrap();
491
492        assert!(folder.join(".git").is_dir());
493    }
494
495    #[test]
496    fn a_mirror_whose_info_cannot_be_written_is_removed() {
497        let tmp = tempfile::tempdir().unwrap();
498        let dest = tmp.path().join("mirror");
499        // A directory where the file goes makes the write fail.
500        std::fs::create_dir_all(dest.join(".info.json")).unwrap();
501
502        let outcome = keep_with_info(&dest, &opts(CloneShape::Mirror));
503
504        assert!(matches!(outcome, Err(CloneError::Info(_))), "{outcome:?}");
505        assert!(!dest.exists());
506    }
507}
fluidattacks_core_git/clone/mod.rs

fluidattacks_core_git/clone/
mod.rs
