fluidattacks_core_git/clone/
mod.rs

1pub mod codecommit;
2pub mod https;
3pub mod ssh;
4
5pub(crate) mod reason;
6
7pub use https::build_https_config_args;
8pub use https::format_https_url;
9pub use reason::hide;
10pub use ssh::setup_ssh_env;
11
12use anyhow::{bail, Context, Result};
13use serde_json::json;
14use std::ffi::OsString;
15use std::fs;
16use std::path::{Path, PathBuf};
17use tokio::process::Command;
18use tracing::{info, warn};
19
20use crate::errors::{CloneError, FileError, HttpsAccessError};
21use crate::types::{CredentialKind, Credentials, GitRoot};
22use https::HttpsAccess;
23
24/// Which refs a clone takes, and what it leaves on disk.
25#[derive(Clone, Copy, Debug, PartialEq, Eq)]
26pub enum CloneShape {
27    /// The named branch and only it, with a working tree.
28    SingleBranch,
29    /// Every ref the remote advertises, as a bare mirror.
30    Mirror,
31    /// Every branch head — `refs/heads/*` plus `HEAD` — as a bare repo:
32    /// no tags, no hidden refs, no mirror config remnants.
33    BareHeads,
34}
35
36pub struct CloneOpts {
37    pub group_name: String,
38    pub nickname: String,
39    pub repo_url: String,
40    pub branch: String,
41    pub credentials: Option<Credentials>,
42    pub follow_redirects: bool,
43    pub skip_existing: bool,
44    pub shape: CloneShape,
45}
46
47impl CloneOpts {
48    #[must_use]
49    pub fn from_root(
50        group: &str,
51        root: &GitRoot,
52        credentials: Option<Credentials>,
53        skip_existing: bool,
54        shape: CloneShape,
55    ) -> Self {
56        Self {
57            group_name: group.to_owned(),
58            nickname: root.nickname.clone(),
59            repo_url: root.url.clone(),
60            branch: root.branch.clone(),
61            credentials,
62            follow_redirects: root.follow_redirects(),
63            skip_existing,
64            shape,
65        }
66    }
67}
68
69async fn dispatch_clone(opts: &CloneOpts, dest: &Path) -> Result<()> {
70    match &opts.credentials {
71        Some(Credentials {
72            kind: CredentialKind::Ssh { .. },
73        }) => ssh::clone_ssh(opts, dest).await,
74        Some(Credentials {
75            kind: CredentialKind::Https { .. },
76        }) => https::clone_https(opts, dest).await,
77        Some(Credentials {
78            kind: CredentialKind::Token { .. },
79        }) => https::clone_https_token(opts, dest).await,
80        Some(Credentials {
81            kind: CredentialKind::AwsRole { .. },
82        }) => codecommit::clone_codecommit(opts, dest).await,
83        None if opts.repo_url.starts_with("http") => https::clone_https_public(opts, dest).await,
84        _ => bail!("no suitable credentials found for root {:?}", opts.nickname),
85    }
86}
87
88pub async fn clone_root(opts: &CloneOpts) -> Result<()> {
89    let repo_dir = PathBuf::from("groups")
90        .join(&opts.group_name)
91        .join(&opts.nickname);
92
93    if opts.skip_existing {
94        if let Ok(entries) = fs::read_dir(&repo_dir) {
95            if entries.count() > 0 {
96                info!(nickname = opts.nickname, "skipping existing repo");
97                return Ok(());
98            }
99        }
100    }
101
102    // Use tempdir_in targeting repo_dir's parent to avoid cross-filesystem rename (EXDEV)
103    let tmp_parent = repo_dir.parent().unwrap_or_else(|| Path::new("."));
104    fs::create_dir_all(tmp_parent)
105        .with_context(|| format!("creating parent dir {}", tmp_parent.display()))?;
106    let tmp_dir = tempfile::tempdir_in(tmp_parent).context("creating temp dir")?;
107    let clone_dest = tmp_dir.path().join(&opts.nickname);
108
109    dispatch_clone(opts, &clone_dest)
110        .await
111        .with_context(|| format!("cloning {:?}", opts.nickname))?;
112
113    if opts.shape == CloneShape::Mirror {
114        convert_bare_to_worktree(&clone_dest)
115            .await
116            .with_context(|| format!("converting mirror to worktree for {:?}", opts.nickname))?;
117    }
118
119    if repo_dir.exists() {
120        fs::remove_dir_all(&repo_dir)
121            .with_context(|| format!("removing old repo dir {}", repo_dir.display()))?;
122    }
123    fs::rename(&clone_dest, &repo_dir)
124        .with_context(|| format!("moving cloned repo to {}", repo_dir.display()))?;
125
126    log_clone_result(&repo_dir, opts).await;
127
128    if opts.shape == CloneShape::Mirror {
129        if let Err(e) = write_mirror_info(&repo_dir, opts) {
130            warn!(error = %e, "failed to write .info.json");
131        }
132    }
133
134    Ok(())
135}
136
137async fn convert_bare_to_worktree(clone_dest: &Path) -> Result<()> {
138    let bare_tmp = clone_dest.with_extension("bare");
139    fs::rename(clone_dest, &bare_tmp)
140        .with_context(|| format!("renaming bare repo to {}", bare_tmp.display()))?;
141    fs::create_dir_all(clone_dest)
142        .with_context(|| format!("creating worktree dir {}", clone_dest.display()))?;
143    let git_dir = clone_dest.join(".git");
144    fs::rename(&bare_tmp, &git_dir)
145        .with_context(|| format!("moving bare contents to {}", git_dir.display()))?;
146
147    let dest_str = clone_dest.to_string_lossy().to_string();
148
149    let config_out = Command::new("git")
150        .args(["-C", &dest_str, "config", "core.bare", "false"])
151        .output()
152        .await
153        .context("running git config core.bare false")?;
154    if !config_out.status.success() {
155        bail!(
156            "git config core.bare false failed: {}",
157            String::from_utf8_lossy(&config_out.stderr)
158        );
159    }
160
161    let reset_out = Command::new("git")
162        .args(["-C", &dest_str, "reset", "--hard", "HEAD"])
163        .output()
164        .await
165        .context("running git reset --hard HEAD")?;
166    if !reset_out.status.success() {
167        bail!(
168            "git reset --hard HEAD failed: {}",
169            String::from_utf8_lossy(&reset_out.stderr)
170        );
171    }
172
173    Ok(())
174}
175
176async fn log_clone_result(repo_dir: &Path, opts: &CloneOpts) {
177    let label = match opts.shape {
178        CloneShape::Mirror => "cloned (mirror)",
179        CloneShape::BareHeads => "cloned (bare heads)",
180        CloneShape::SingleBranch => "cloned",
181    };
182    let repo_dir_str = repo_dir.to_string_lossy().to_string();
183    match Command::new("git")
184        .args(["-C", &repo_dir_str, "log", "-1", "--format=%H %ai"])
185        .output()
186        .await
187    {
188        Ok(output) if output.status.success() => {
189            let head = String::from_utf8_lossy(&output.stdout).trim().to_owned();
190            info!(nickname = opts.nickname, head = head, label);
191        }
192        _ => {
193            info!(nickname = opts.nickname, dir = %repo_dir.display(), label);
194        }
195    }
196}
197
198fn write_mirror_info(repo_dir: &Path, opts: &CloneOpts) -> Result<(), FileError> {
199    let info = json!({
200        "fluid_branch": opts.branch,
201        "repo": opts.repo_url,
202    });
203    let path = repo_dir.join(".info.json");
204    fs::write(&path, format!("{info:#}")).map_err(|source| FileError {
205        action: "writing",
206        path,
207        source,
208    })
209}
210
211/// git creates whatever part of `temp_dir` is missing.
212#[expect(
213    clippy::too_many_arguments,
214    reason = "what the python https_clone, ssh_clone and call_codecommit_clone take, the credential in one"
215)]
216pub async fn clone_to_temp(
217    temp_dir: &Path,
218    url: &str,
219    branch: &str,
220    credential: Option<&CredentialKind>,
221    follow_redirects: bool,
222    mirror: bool,
223) -> Result<PathBuf, CloneError> {
224    let nickname = uuid::Uuid::new_v4().to_string();
225    let dest = temp_dir.join(&nickname);
226    let opts = CloneOpts {
227        group_name: String::new(),
228        nickname,
229        repo_url: url.to_owned(),
230        branch: branch.to_owned(),
231        credentials: None,
232        follow_redirects,
233        skip_existing: false,
234        shape: if mirror {
235            CloneShape::Mirror
236        } else {
237            CloneShape::SingleBranch
238        },
239    };
240    let cloned = match credential {
241        Some(CredentialKind::Ssh { key }) => ssh::run_git_clone(&opts, &dest, key).await,
242        Some(CredentialKind::AwsRole {
243            arn,
244            external_id,
245            via,
246        }) => codecommit::run_git_clone(&opts, &dest, arn, external_id, via.as_deref()).await,
247        // git takes any url, an scp-like one over ssh with whatever keys this machine has.
248        None if !has_http_scheme(url) => return Err(HttpsAccessError::NotHttp.into()),
249        _ => {
250            let access = HttpsAccess::new(url, credential)?;
251            Ok(https::run_git_clone(&opts, &dest, &access).await?)
252        }
253    };
254    cloned.inspect_err(|_| {
255        let _ = fs::remove_dir_all(&dest);
256    })?;
257    log_clone_result(&dest, &opts).await;
258    if mirror {
259        keep_with_info(&dest, &opts)?;
260    }
261    Ok(dest)
262}
263
264fn has_http_scheme(url: &str) -> bool {
265    url.split_once("://")
266        .is_some_and(|(scheme, _)| matches!(scheme, "http" | "https"))
267}
268
269fn keep_with_info(dest: &Path, opts: &CloneOpts) -> Result<(), CloneError> {
270    // On an error the caller never gets the path, so nobody would remove this clone.
271    write_mirror_info(dest, opts).map_err(|error| {
272        let _ = fs::remove_dir_all(dest);
273        error.into()
274    })
275}
276
277#[must_use]
278pub fn build_clone_args(
279    opts: &CloneOpts,
280    repo_url: &str,
281    dest: &Path,
282    extra_config: &[String],
283) -> Vec<OsString> {
284    let mut args = vec!["-c".to_owned(), "core.symlinks=false".to_owned()];
285
286    args.extend(extra_config.iter().cloned());
287
288    args.push("clone".to_owned());
289
290    match opts.shape {
291        CloneShape::Mirror => args.push("--mirror".to_owned()),
292        // Every branch head and nothing else: unlike --mirror this skips
293        // refs/pull, refs/merge-requests and keep-around refs, and writes no
294        // mirror refspec into the clone's config.
295        CloneShape::BareHeads => args.extend(["--bare".to_owned(), "--no-tags".to_owned()]),
296        // An empty branch is passed on for git to refuse, not read as the default one.
297        CloneShape::SingleBranch => args.extend([
298            "--branch".to_owned(),
299            opts.branch.clone(),
300            "--single-branch".to_owned(),
301        ]),
302    }
303
304    args.extend(["--".to_owned(), repo_url.to_owned()]);
305
306    let mut args: Vec<OsString> = args.into_iter().map(OsString::from).collect();
307    // A path need not be UTF-8, so git gets its bytes.
308    args.push(dest.as_os_str().to_owned());
309    args
310}
311
312pub(crate) fn redirect_config(follow_redirects: bool) -> [String; 2] {
313    [
314        "-c".to_owned(),
315        format!("http.followRedirects={follow_redirects}"),
316    ]
317}
318
319#[cfg(test)]
320mod tests {
321    use std::ffi::{OsStr, OsString};
322    use std::io::{BufRead, BufReader, Write};
323    use std::net::TcpListener;
324    use std::os::unix::ffi::OsStrExt;
325    use std::path::Path;
326    use std::process::Command;
327
328    use super::{build_clone_args, clone_to_temp, keep_with_info, CloneOpts, CloneShape};
329    use crate::errors::{CloneError, HttpsAccessError};
330
331    pub(super) fn opts(shape: CloneShape) -> CloneOpts {
332        CloneOpts {
333            group_name: "a-group".to_owned(),
334            nickname: "a_root".to_owned(),
335            repo_url: "https://gitlab.com/org/repo".to_owned(),
336            branch: "trunk".to_owned(),
337            credentials: None,
338            follow_redirects: false,
339            skip_existing: false,
340            shape,
341        }
342    }
343
344    fn shape_args(shape: CloneShape) -> Vec<OsString> {
345        let opts = opts(shape);
346
347        // Between "clone" and "--" sits exactly the shape.
348        build_clone_args(&opts, &opts.repo_url, Path::new("/tmp/a_root"), &[])
349            .iter()
350            .skip_while(|arg| *arg != "clone")
351            .skip(1)
352            .take_while(|arg| *arg != "--")
353            .cloned()
354            .collect()
355    }
356
357    fn run_git(dir: &Path, args: &[&str]) -> String {
358        let output = Command::new("git")
359            .arg("-C")
360            .arg(dir)
361            .args([
362                "-c",
363                "user.email=test@example.com",
364                "-c",
365                "user.name=Test",
366                "-c",
367                "commit.gpgsign=false",
368                "-c",
369                "tag.gpgsign=false",
370            ])
371            .args(args)
372            .output()
373            .expect("git runs");
374        assert!(
375            output.status.success(),
376            "git {args:?} failed: {}",
377            String::from_utf8_lossy(&output.stderr)
378        );
379        String::from_utf8_lossy(&output.stdout).trim().to_owned()
380    }
381
382    #[test]
383    fn a_clone_never_writes_a_symbolic_link() {
384        let opts = opts(CloneShape::SingleBranch);
385        let args = build_clone_args(&opts, &opts.repo_url, Path::new("/tmp/a_root"), &[]);
386
387        assert!(args
388            .windows(2)
389            .any(|pair| pair == ["-c", "core.symlinks=false"]));
390    }
391
392    #[test]
393    fn a_destination_that_is_not_utf8_reaches_git_as_its_bytes() {
394        let opts = opts(CloneShape::SingleBranch);
395        let dest = Path::new(OsStr::from_bytes(b"/tmp/caf\xe9"));
396
397        let args = build_clone_args(&opts, &opts.repo_url, dest, &[]);
398
399        assert_eq!(args.last().map(OsString::as_os_str), Some(dest.as_os_str()));
400    }
401
402    #[test]
403    fn a_branch_clone_is_single_branch() {
404        assert_eq!(
405            shape_args(CloneShape::SingleBranch),
406            ["--branch", "trunk", "--single-branch"]
407        );
408    }
409
410    #[test]
411    fn an_empty_branch_is_asked_for_instead_of_the_default_one() {
412        let opts = CloneOpts {
413            branch: String::new(),
414            ..opts(CloneShape::SingleBranch)
415        };
416        let args = build_clone_args(&opts, &opts.repo_url, Path::new("/tmp/a_root"), &[]);
417
418        assert!(args.windows(2).any(|pair| pair == ["--branch", ""]));
419    }
420
421    #[test]
422    fn a_mirror_clone_takes_every_ref() {
423        assert_eq!(shape_args(CloneShape::Mirror), ["--mirror"]);
424    }
425
426    #[test]
427    fn a_bare_heads_clone_takes_heads_without_tags() {
428        assert_eq!(shape_args(CloneShape::BareHeads), ["--bare", "--no-tags"]);
429    }
430
431    #[test]
432    fn a_bare_heads_clone_holds_exactly_the_branch_heads() {
433        let tmp = tempfile::tempdir().expect("tempdir");
434        let source = tmp.path().join("source");
435        std::fs::create_dir(&source).expect("mkdir");
436        run_git(&source, &["init", "--initial-branch=main"]);
437        std::fs::write(source.join("file.txt"), "content\n").expect("write");
438        run_git(&source, &["add", "file.txt"]);
439        run_git(&source, &["commit", "-m", "first"]);
440        run_git(&source, &["branch", "feature"]);
441        run_git(&source, &["tag", "v1"]);
442        // The kind of hidden ref --mirror would drag in.
443        run_git(&source, &["update-ref", "refs/pull/1/head", "HEAD"]);
444
445        let dest = tmp.path().join("dest");
446        let opts = opts(CloneShape::BareHeads);
447        let args = build_clone_args(&opts, &source.to_string_lossy(), &dest, &[]);
448        let clone = Command::new("git").args(&args).output().expect("git runs");
449        assert!(
450            clone.status.success(),
451            "clone failed: {}",
452            String::from_utf8_lossy(&clone.stderr)
453        );
454
455        let refs = run_git(&dest, &["for-each-ref", "--format=%(refname)"]);
456        let mut held: Vec<&str> = refs.lines().collect();
457        held.sort_unstable();
458        assert_eq!(held, ["refs/heads/feature", "refs/heads/main"]);
459        assert_eq!(run_git(&dest, &["symbolic-ref", "HEAD"]), "refs/heads/main");
460        assert_eq!(run_git(&dest, &["config", "core.bare"]), "true");
461        // No mirror remnants: neither the flag nor a fetch refspec.
462        for key in ["remote.origin.mirror", "remote.origin.fetch"] {
463            let lookup = Command::new("git")
464                .args(["-C", &dest.to_string_lossy(), "config", "--get", key])
465                .output()
466                .expect("git runs");
467            assert!(!lookup.status.success(), "{key} is set");
468        }
469    }
470
471    pub(super) fn source(tmp: &Path) -> String {
472        let work = tmp.join("work");
473        std::fs::create_dir(&work).unwrap();
474        run_git(&work, &["init", "--quiet", "--initial-branch=main"]);
475        run_git(&work, &["commit", "--quiet", "--allow-empty", "-m", "one"]);
476        format!("file://{}", work.display())
477    }
478
479    // `source` over git's dumb http protocol: a static file server on a bare copy.
480    fn served(tmp: &Path) -> String {
481        let bare = tmp.join("served.git");
482        let cloned = Command::new("git")
483            .args(["clone", "--quiet", "--bare", &source(tmp)])
484            .arg(&bare)
485            .status()
486            .unwrap();
487        assert!(cloned.success());
488        run_git(&bare, &["update-server-info"]);
489        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
490        let port = listener.local_addr().unwrap().port();
491        std::thread::spawn(move || {
492            for mut stream in listener.incoming().flatten() {
493                let mut lines = BufReader::new(&stream).lines();
494                let request = lines.next().and_then(Result::ok).unwrap_or_default();
495                // Unread headers left behind make the close a reset that git may see first.
496                lines
497                    .map_while(Result::ok)
498                    .take_while(|line| !line.is_empty())
499                    .for_each(drop);
500                let _ = stream.write_all(&file_answer(&bare, &request));
501            }
502        });
503        format!("http://127.0.0.1:{port}/")
504    }
505
506    // The file a request line asks for under `root`, query left out, or a 404.
507    fn file_answer(root: &Path, request: &str) -> Vec<u8> {
508        let path = request.split(' ').nth(1).unwrap_or_default();
509        let file = path.split('?').next().unwrap_or_default();
510        std::fs::read(root.join(file.trim_start_matches('/'))).map_or_else(
511            |_| {
512                b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec()
513            },
514            |body| {
515                let head = format!(
516                    "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
517                    body.len()
518                );
519                [head.into_bytes(), body].concat()
520            },
521        )
522    }
523
524    #[tokio::test]
525    async fn git_creates_a_missing_temp_dir() {
526        let tmp = tempfile::tempdir().unwrap();
527        let url = served(tmp.path());
528        let temp_dir = tmp.path().join("missing/clones");
529
530        let folder = clone_to_temp(&temp_dir, &url, "main", None, false, false)
531            .await
532            .unwrap();
533
534        assert_eq!(folder.parent(), Some(temp_dir.as_path()));
535    }
536
537    #[tokio::test]
538    async fn a_url_without_a_credential_goes_to_git_as_given() {
539        let tmp = tempfile::tempdir().unwrap();
540        let url = served(tmp.path());
541
542        let folder = clone_to_temp(tmp.path(), &url, "main", None, false, false)
543            .await
544            .unwrap();
545
546        assert!(folder.join(".git").is_dir());
547    }
548
549    #[tokio::test]
550    async fn a_url_that_is_not_http_reaches_no_git_without_a_credential() {
551        let tmp = tempfile::tempdir().unwrap();
552        // git would clone this one.
553        let url = source(tmp.path());
554        let temp_dir = tmp.path().join("clones");
555
556        let outcome = clone_to_temp(&temp_dir, &url, "main", None, false, false).await;
557
558        assert!(
559            matches!(outcome, Err(CloneError::Access(HttpsAccessError::NotHttp))),
560            "{outcome:?}"
561        );
562        assert!(!temp_dir.exists());
563    }
564
565    #[test]
566    fn a_mirror_whose_info_cannot_be_written_is_removed() {
567        let tmp = tempfile::tempdir().unwrap();
568        let dest = tmp.path().join("mirror");
569        // A directory where the file goes makes the write fail.
570        std::fs::create_dir_all(dest.join(".info.json")).unwrap();
571
572        let outcome = keep_with_info(&dest, &opts(CloneShape::Mirror));
573
574        assert!(matches!(outcome, Err(CloneError::Info(_))), "{outcome:?}");
575        assert!(!dest.exists());
576    }
577}
fluidattacks_core_git/clone/mod.rs

fluidattacks_core_git/clone/
mod.rs
