fluidattacks_core_git/clone/
mod.rs

1pub mod codecommit;
2pub mod https;
3pub mod ssh;
4
5pub(crate) mod reason;
6
7pub use https::build_https_config_args;
8pub use https::format_https_url;
9pub use reason::hide;
10pub use ssh::setup_ssh_env;
11
12use anyhow::{bail, Context, Result};
13use serde_json::json;
14use std::ffi::OsString;
15use std::fs;
16use std::path::{Path, PathBuf};
17use tokio::process::Command;
18use tracing::{info, warn};
19
20use crate::errors::{CloneError, FileError, HttpsAccessError};
21use crate::types::{CredentialKind, Credentials, GitRoot};
22use https::HttpsAccess;
23
24/// Which refs a clone takes, and what it leaves on disk.
25#[derive(Clone, Copy, Debug, PartialEq, Eq)]
26pub enum CloneShape {
27    /// The named branch and only it, with a working tree.
28    SingleBranch,
29    /// Every ref the remote advertises, as a bare mirror.
30    Mirror,
31    /// Every branch head — `refs/heads/*` plus `HEAD` — as a bare repo:
32    /// no tags, no hidden refs, no mirror config remnants.
33    BareHeads,
34}
35
36pub struct CloneOpts {
37    pub group_name: String,
38    pub nickname: String,
39    pub repo_url: String,
40    pub branch: String,
41    pub credentials: Option<Credentials>,
42    pub follow_redirects: bool,
43    pub skip_existing: bool,
44    pub shape: CloneShape,
45}
46
47impl CloneOpts {
48    #[must_use]
49    pub fn from_root(
50        group: &str,
51        root: &GitRoot,
52        credentials: Option<Credentials>,
53        skip_existing: bool,
54        shape: CloneShape,
55    ) -> Self {
56        Self {
57            group_name: group.to_owned(),
58            nickname: root.nickname.clone(),
59            repo_url: root.url.clone(),
60            branch: root.branch.clone(),
61            credentials,
62            follow_redirects: root.follow_redirects(),
63            skip_existing,
64            shape,
65        }
66    }
67}
68
69async fn dispatch_clone(opts: &CloneOpts, dest: &Path) -> Result<()> {
70    match &opts.credentials {
71        Some(Credentials {
72            kind: CredentialKind::Ssh { .. },
73        }) => ssh::clone_ssh(opts, dest).await,
74        Some(Credentials {
75            kind: CredentialKind::Https { .. },
76        }) => https::clone_https(opts, dest).await,
77        Some(Credentials {
78            kind: CredentialKind::Token { .. },
79        }) => https::clone_https_token(opts, dest).await,
80        Some(Credentials {
81            kind: CredentialKind::AwsRole { .. },
82        }) => codecommit::clone_codecommit(opts, dest).await,
83        None if opts.repo_url.starts_with("http") => https::clone_https_public(opts, dest).await,
84        _ => bail!("no suitable credentials found for root {:?}", opts.nickname),
85    }
86}
87
88pub async fn clone_root(opts: &CloneOpts) -> Result<()> {
89    let repo_dir = PathBuf::from("groups")
90        .join(&opts.group_name)
91        .join(&opts.nickname);
92
93    if opts.skip_existing {
94        if let Ok(entries) = fs::read_dir(&repo_dir) {
95            if entries.count() > 0 {
96                info!(nickname = opts.nickname, "skipping existing repo");
97                return Ok(());
98            }
99        }
100    }
101
102    // Use tempdir_in targeting repo_dir's parent to avoid cross-filesystem rename (EXDEV)
103    let tmp_parent = repo_dir.parent().unwrap_or_else(|| Path::new("."));
104    fs::create_dir_all(tmp_parent)
105        .with_context(|| format!("creating parent dir {}", tmp_parent.display()))?;
106    let tmp_dir = tempfile::tempdir_in(tmp_parent).context("creating temp dir")?;
107    let clone_dest = tmp_dir.path().join(&opts.nickname);
108
109    dispatch_clone(opts, &clone_dest)
110        .await
111        .with_context(|| format!("cloning {:?}", opts.nickname))?;
112
113    if opts.shape == CloneShape::Mirror {
114        convert_bare_to_worktree(&clone_dest)
115            .await
116            .with_context(|| format!("converting mirror to worktree for {:?}", opts.nickname))?;
117    }
118
119    if repo_dir.exists() {
120        fs::remove_dir_all(&repo_dir)
121            .with_context(|| format!("removing old repo dir {}", repo_dir.display()))?;
122    }
123    fs::rename(&clone_dest, &repo_dir)
124        .with_context(|| format!("moving cloned repo to {}", repo_dir.display()))?;
125
126    log_clone_result(&repo_dir, opts).await;
127
128    if opts.shape == CloneShape::Mirror {
129        if let Err(e) = write_mirror_info(&repo_dir, opts) {
130            warn!(error = %e, "failed to write .info.json");
131        }
132    }
133
134    Ok(())
135}
136
137async fn convert_bare_to_worktree(clone_dest: &Path) -> Result<()> {
138    let bare_tmp = clone_dest.with_extension("bare");
139    fs::rename(clone_dest, &bare_tmp)
140        .with_context(|| format!("renaming bare repo to {}", bare_tmp.display()))?;
141    fs::create_dir_all(clone_dest)
142        .with_context(|| format!("creating worktree dir {}", clone_dest.display()))?;
143    let git_dir = clone_dest.join(".git");
144    fs::rename(&bare_tmp, &git_dir)
145        .with_context(|| format!("moving bare contents to {}", git_dir.display()))?;
146
147    let dest_str = clone_dest.to_string_lossy().to_string();
148
149    let config_out = Command::new("git")
150        .args(["-C", &dest_str, "config", "core.bare", "false"])
151        .output()
152        .await
153        .context("running git config core.bare false")?;
154    if !config_out.status.success() {
155        bail!(
156            "git config core.bare false failed: {}",
157            String::from_utf8_lossy(&config_out.stderr)
158        );
159    }
160
161    let reset_out = Command::new("git")
162        .args(["-C", &dest_str, "reset", "--hard", "HEAD"])
163        .output()
164        .await
165        .context("running git reset --hard HEAD")?;
166    if !reset_out.status.success() {
167        bail!(
168            "git reset --hard HEAD failed: {}",
169            String::from_utf8_lossy(&reset_out.stderr)
170        );
171    }
172
173    Ok(())
174}
175
176async fn log_clone_result(repo_dir: &Path, opts: &CloneOpts) {
177    let label = match opts.shape {
178        CloneShape::Mirror => "cloned (mirror)",
179        CloneShape::BareHeads => "cloned (bare heads)",
180        CloneShape::SingleBranch => "cloned",
181    };
182    let repo_dir_str = repo_dir.to_string_lossy().to_string();
183    match Command::new("git")
184        .args(["-C", &repo_dir_str, "log", "-1", "--format=%H %ai"])
185        .output()
186        .await
187    {
188        Ok(output) if output.status.success() => {
189            let head = String::from_utf8_lossy(&output.stdout).trim().to_owned();
190            info!(nickname = opts.nickname, head = head, label);
191        }
192        _ => {
193            info!(nickname = opts.nickname, dir = %repo_dir.display(), label);
194        }
195    }
196}
197
198fn write_mirror_info(repo_dir: &Path, opts: &CloneOpts) -> Result<(), FileError> {
199    let info = json!({
200        "fluid_branch": opts.branch,
201        "repo": opts.repo_url,
202    });
203    let path = repo_dir.join(".info.json");
204    fs::write(&path, format!("{info:#}")).map_err(|source| FileError {
205        action: "writing",
206        path,
207        source,
208    })
209}
210
211/// git creates whatever part of `temp_dir` is missing.
212#[expect(
213    clippy::too_many_arguments,
214    reason = "what the python https_clone, ssh_clone and call_codecommit_clone take, the credential in one"
215)]
216pub async fn clone_to_temp(
217    temp_dir: &Path,
218    url: &str,
219    branch: &str,
220    credential: Option<&CredentialKind>,
221    follow_redirects: bool,
222    mirror: bool,
223) -> Result<PathBuf, CloneError> {
224    let nickname = uuid::Uuid::new_v4().to_string();
225    let dest = temp_dir.join(&nickname);
226    let opts = CloneOpts {
227        group_name: String::new(),
228        nickname,
229        repo_url: url.to_owned(),
230        branch: branch.to_owned(),
231        credentials: None,
232        follow_redirects,
233        skip_existing: false,
234        shape: if mirror {
235            CloneShape::Mirror
236        } else {
237            CloneShape::SingleBranch
238        },
239    };
240    let cloned = match credential {
241        Some(CredentialKind::Ssh { key }) => ssh::run_git_clone(&opts, &dest, key).await,
242        Some(CredentialKind::AwsRole {
243            arn,
244            external_id,
245            via,
246        }) => codecommit::run_git_clone(&opts, &dest, arn, external_id, via.as_deref()).await,
247        // git takes any url, an scp-like one over ssh with whatever keys this machine has.
248        None if !has_http_scheme(url) => return Err(HttpsAccessError::NotHttp.into()),
249        _ => {
250            let access = HttpsAccess::new(url, credential)?;
251            Ok(https::run_git_clone(&opts, &dest, &access).await?)
252        }
253    };
254    cloned.inspect_err(|_| {
255        let _ = fs::remove_dir_all(&dest);
256    })?;
257    log_clone_result(&dest, &opts).await;
258    if mirror {
259        keep_with_info(&dest, &opts)?;
260    }
261    Ok(dest)
262}
263
264pub(crate) fn has_http_scheme(url: &str) -> bool {
265    url.split_once("://")
266        .is_some_and(|(scheme, _)| matches!(scheme, "http" | "https"))
267}
268
269fn keep_with_info(dest: &Path, opts: &CloneOpts) -> Result<(), CloneError> {
270    // On an error the caller never gets the path, so nobody would remove this clone.
271    write_mirror_info(dest, opts).map_err(|error| {
272        let _ = fs::remove_dir_all(dest);
273        error.into()
274    })
275}
276
277#[must_use]
278pub fn build_clone_args(
279    opts: &CloneOpts,
280    repo_url: &str,
281    dest: &Path,
282    extra_config: &[String],
283) -> Vec<OsString> {
284    let mut args = vec!["-c".to_owned(), "core.symlinks=false".to_owned()];
285
286    args.extend(extra_config.iter().cloned());
287
288    args.push("clone".to_owned());
289
290    match opts.shape {
291        CloneShape::Mirror => args.push("--mirror".to_owned()),
292        // Every branch head and nothing else: unlike --mirror this skips
293        // refs/pull, refs/merge-requests and keep-around refs, and writes no
294        // mirror refspec into the clone's config.
295        CloneShape::BareHeads => args.extend(["--bare".to_owned(), "--no-tags".to_owned()]),
296        // An empty branch is passed on for git to refuse, not read as the default one.
297        CloneShape::SingleBranch => args.extend([
298            "--branch".to_owned(),
299            opts.branch.clone(),
300            "--single-branch".to_owned(),
301        ]),
302    }
303
304    args.extend(["--".to_owned(), repo_url.to_owned()]);
305
306    let mut args: Vec<OsString> = args.into_iter().map(OsString::from).collect();
307    // A path need not be UTF-8, so git gets its bytes.
308    args.push(dest.as_os_str().to_owned());
309    args
310}
311
312pub(crate) fn redirect_config(follow_redirects: bool) -> [String; 2] {
313    [
314        "-c".to_owned(),
315        format!("http.followRedirects={follow_redirects}"),
316    ]
317}
318
319pub(crate) fn stall_config() -> [String; 4] {
320    [
321        "-c".to_owned(),
322        "http.lowSpeedLimit=1000".to_owned(),
323        "-c".to_owned(),
324        "http.lowSpeedTime=600".to_owned(),
325    ]
326}
327
328#[cfg(test)]
329mod tests {
330    use std::ffi::{OsStr, OsString};
331    use std::io::{BufRead, BufReader, Write};
332    use std::net::TcpListener;
333    use std::os::unix::ffi::OsStrExt;
334    use std::path::Path;
335    use std::process::Command;
336
337    use super::{build_clone_args, clone_to_temp, keep_with_info, CloneOpts, CloneShape};
338    use crate::errors::{CloneError, HttpsAccessError};
339
340    pub(super) fn opts(shape: CloneShape) -> CloneOpts {
341        CloneOpts {
342            group_name: "a-group".to_owned(),
343            nickname: "a_root".to_owned(),
344            repo_url: "https://gitlab.com/org/repo".to_owned(),
345            branch: "trunk".to_owned(),
346            credentials: None,
347            follow_redirects: false,
348            skip_existing: false,
349            shape,
350        }
351    }
352
353    fn shape_args(shape: CloneShape) -> Vec<OsString> {
354        let opts = opts(shape);
355
356        // Between "clone" and "--" sits exactly the shape.
357        build_clone_args(&opts, &opts.repo_url, Path::new("/tmp/a_root"), &[])
358            .iter()
359            .skip_while(|arg| *arg != "clone")
360            .skip(1)
361            .take_while(|arg| *arg != "--")
362            .cloned()
363            .collect()
364    }
365
366    fn run_git(dir: &Path, args: &[&str]) -> String {
367        let output = Command::new("git")
368            .arg("-C")
369            .arg(dir)
370            .args([
371                "-c",
372                "user.email=test@example.com",
373                "-c",
374                "user.name=Test",
375                "-c",
376                "commit.gpgsign=false",
377                "-c",
378                "tag.gpgsign=false",
379            ])
380            .args(args)
381            .output()
382            .expect("git runs");
383        assert!(
384            output.status.success(),
385            "git {args:?} failed: {}",
386            String::from_utf8_lossy(&output.stderr)
387        );
388        String::from_utf8_lossy(&output.stdout).trim().to_owned()
389    }
390
391    #[test]
392    fn a_clone_never_writes_a_symbolic_link() {
393        let opts = opts(CloneShape::SingleBranch);
394        let args = build_clone_args(&opts, &opts.repo_url, Path::new("/tmp/a_root"), &[]);
395
396        assert!(args
397            .windows(2)
398            .any(|pair| pair == ["-c", "core.symlinks=false"]));
399    }
400
401    #[test]
402    fn a_destination_that_is_not_utf8_reaches_git_as_its_bytes() {
403        let opts = opts(CloneShape::SingleBranch);
404        let dest = Path::new(OsStr::from_bytes(b"/tmp/caf\xe9"));
405
406        let args = build_clone_args(&opts, &opts.repo_url, dest, &[]);
407
408        assert_eq!(args.last().map(OsString::as_os_str), Some(dest.as_os_str()));
409    }
410
411    #[test]
412    fn a_branch_clone_is_single_branch() {
413        assert_eq!(
414            shape_args(CloneShape::SingleBranch),
415            ["--branch", "trunk", "--single-branch"]
416        );
417    }
418
419    #[test]
420    fn an_empty_branch_is_asked_for_instead_of_the_default_one() {
421        let opts = CloneOpts {
422            branch: String::new(),
423            ..opts(CloneShape::SingleBranch)
424        };
425        let args = build_clone_args(&opts, &opts.repo_url, Path::new("/tmp/a_root"), &[]);
426
427        assert!(args.windows(2).any(|pair| pair == ["--branch", ""]));
428    }
429
430    #[test]
431    fn a_mirror_clone_takes_every_ref() {
432        assert_eq!(shape_args(CloneShape::Mirror), ["--mirror"]);
433    }
434
435    #[test]
436    fn a_bare_heads_clone_takes_heads_without_tags() {
437        assert_eq!(shape_args(CloneShape::BareHeads), ["--bare", "--no-tags"]);
438    }
439
440    #[test]
441    fn a_bare_heads_clone_holds_exactly_the_branch_heads() {
442        let tmp = tempfile::tempdir().expect("tempdir");
443        let source = tmp.path().join("source");
444        std::fs::create_dir(&source).expect("mkdir");
445        run_git(&source, &["init", "--initial-branch=main"]);
446        std::fs::write(source.join("file.txt"), "content\n").expect("write");
447        run_git(&source, &["add", "file.txt"]);
448        run_git(&source, &["commit", "-m", "first"]);
449        run_git(&source, &["branch", "feature"]);
450        run_git(&source, &["tag", "v1"]);
451        // The kind of hidden ref --mirror would drag in.
452        run_git(&source, &["update-ref", "refs/pull/1/head", "HEAD"]);
453
454        let dest = tmp.path().join("dest");
455        let opts = opts(CloneShape::BareHeads);
456        let args = build_clone_args(&opts, &source.to_string_lossy(), &dest, &[]);
457        let clone = Command::new("git").args(&args).output().expect("git runs");
458        assert!(
459            clone.status.success(),
460            "clone failed: {}",
461            String::from_utf8_lossy(&clone.stderr)
462        );
463
464        let refs = run_git(&dest, &["for-each-ref", "--format=%(refname)"]);
465        let mut held: Vec<&str> = refs.lines().collect();
466        held.sort_unstable();
467        assert_eq!(held, ["refs/heads/feature", "refs/heads/main"]);
468        assert_eq!(run_git(&dest, &["symbolic-ref", "HEAD"]), "refs/heads/main");
469        assert_eq!(run_git(&dest, &["config", "core.bare"]), "true");
470        // No mirror remnants: neither the flag nor a fetch refspec.
471        for key in ["remote.origin.mirror", "remote.origin.fetch"] {
472            let lookup = Command::new("git")
473                .args(["-C", &dest.to_string_lossy(), "config", "--get", key])
474                .output()
475                .expect("git runs");
476            assert!(!lookup.status.success(), "{key} is set");
477        }
478    }
479
480    pub(super) fn source(tmp: &Path) -> String {
481        let work = tmp.join("work");
482        std::fs::create_dir(&work).unwrap();
483        run_git(&work, &["init", "--quiet", "--initial-branch=main"]);
484        run_git(&work, &["commit", "--quiet", "--allow-empty", "-m", "one"]);
485        format!("file://{}", work.display())
486    }
487
488    // `source` over git's dumb http protocol: a static file server on a bare copy.
489    fn served(tmp: &Path) -> String {
490        let bare = tmp.join("served.git");
491        let cloned = Command::new("git")
492            .args(["clone", "--quiet", "--bare", &source(tmp)])
493            .arg(&bare)
494            .status()
495            .unwrap();
496        assert!(cloned.success());
497        run_git(&bare, &["update-server-info"]);
498        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
499        let port = listener.local_addr().unwrap().port();
500        std::thread::spawn(move || {
501            for mut stream in listener.incoming().flatten() {
502                let mut lines = BufReader::new(&stream).lines();
503                let request = lines.next().and_then(Result::ok).unwrap_or_default();
504                // Unread headers left behind make the close a reset that git may see first.
505                lines
506                    .map_while(Result::ok)
507                    .take_while(|line| !line.is_empty())
508                    .for_each(drop);
509                let _ = stream.write_all(&file_answer(&bare, &request));
510            }
511        });
512        format!("http://127.0.0.1:{port}/")
513    }
514
515    // The file a request line asks for under `root`, query left out, or a 404.
516    fn file_answer(root: &Path, request: &str) -> Vec<u8> {
517        let path = request.split(' ').nth(1).unwrap_or_default();
518        let file = path.split('?').next().unwrap_or_default();
519        std::fs::read(root.join(file.trim_start_matches('/'))).map_or_else(
520            |_| {
521                b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec()
522            },
523            |body| {
524                let head = format!(
525                    "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
526                    body.len()
527                );
528                [head.into_bytes(), body].concat()
529            },
530        )
531    }
532
533    #[tokio::test]
534    async fn git_creates_a_missing_temp_dir() {
535        let tmp = tempfile::tempdir().unwrap();
536        let url = served(tmp.path());
537        let temp_dir = tmp.path().join("missing/clones");
538
539        let folder = clone_to_temp(&temp_dir, &url, "main", None, false, false)
540            .await
541            .unwrap();
542
543        assert_eq!(folder.parent(), Some(temp_dir.as_path()));
544    }
545
546    #[tokio::test]
547    async fn a_url_without_a_credential_goes_to_git_as_given() {
548        let tmp = tempfile::tempdir().unwrap();
549        let url = served(tmp.path());
550
551        let folder = clone_to_temp(tmp.path(), &url, "main", None, false, false)
552            .await
553            .unwrap();
554
555        assert!(folder.join(".git").is_dir());
556    }
557
558    #[tokio::test]
559    async fn a_url_that_is_not_http_reaches_no_git_without_a_credential() {
560        let tmp = tempfile::tempdir().unwrap();
561        // git would clone this one.
562        let url = source(tmp.path());
563        let temp_dir = tmp.path().join("clones");
564
565        let outcome = clone_to_temp(&temp_dir, &url, "main", None, false, false).await;
566
567        assert!(
568            matches!(outcome, Err(CloneError::Access(HttpsAccessError::NotHttp))),
569            "{outcome:?}"
570        );
571        assert!(!temp_dir.exists());
572    }
573
574    #[test]
575    fn a_mirror_whose_info_cannot_be_written_is_removed() {
576        let tmp = tempfile::tempdir().unwrap();
577        let dest = tmp.path().join("mirror");
578        // A directory where the file goes makes the write fail.
579        std::fs::create_dir_all(dest.join(".info.json")).unwrap();
580
581        let outcome = keep_with_info(&dest, &opts(CloneShape::Mirror));
582
583        assert!(matches!(outcome, Err(CloneError::Info(_))), "{outcome:?}");
584        assert!(!dest.exists());
585    }
586}
fluidattacks_core_git/clone/mod.rs

fluidattacks_core_git/clone/
mod.rs
