fluidattacks_core_git/

download.rs

use anyhow::{bail, Context, Result};
use std::io;
use std::num::NonZeroUsize;
use std::path::{Path, PathBuf};
use std::time::Duration;
use tokio::io::{AsyncWriteExt, BufWriter};
use tracing::warn;

use crate::errors::{DownloadError, FetchError};

const TIMEOUTS: Timeouts = Timeouts {
    connect: Duration::from_secs(30),
    total: Duration::from_hours(1),
};
const DEFAULT_DOWNLOAD_BUFFER_SIZE: usize = 64 * 1024;
const MAX_DOWNLOAD_BUFFER_SIZE: usize = 16 * 1024 * 1024;
const TMP_PREFIX: &str = "fluidattacks_";
const ARCHIVE_NAME: &str = "repo.tar.gz";
const GIT_DIR_DATA: [&str; 6] = ["HEAD", "index", "objects", "packed-refs", "refs", "shallow"];
// Inside `objects/`, these point git at object stores elsewhere on disk.
const OBJECT_STORE_POINTERS: [&str; 2] = ["alternates", "http-alternates"];
const TRUSTED_CONFIG: &str = "[core]\nrepositoryformatversion = 0\nbare = false\n";

struct Timeouts {
    connect: Duration,
    total: Duration,
}

fn resolve_buffer_size(buffer_size: Option<NonZeroUsize>) -> usize {
    buffer_size.map_or(DEFAULT_DOWNLOAD_BUFFER_SIZE, |size| {
        size.get().min(MAX_DOWNLOAD_BUFFER_SIZE)
    })
}

fn parse_url(url: &str) -> Result<url::Url, FetchError> {
    let parsed = url::Url::parse(url).map_err(|error| FetchError::InvalidUrl {
        url: url.to_owned(),
        reason: error.to_string(),
    })?;
    if !matches!(parsed.scheme(), "http" | "https") {
        return Err(FetchError::NonHttpUrl(url.to_owned()));
    }
    Ok(parsed)
}

fn request_error(error: reqwest::Error) -> FetchError {
    if error.is_connect() {
        return if error.is_timeout() {
            FetchError::ConnectTimeout(error)
        } else {
            FetchError::Connect(error)
        };
    }
    if error.is_timeout() {
        return FetchError::Timeout(error);
    }
    let mut cause: Option<&(dyn std::error::Error + 'static)> = Some(&error);
    let mut socket = None;
    while let Some(current) = cause {
        if let Some(hyper) = current.downcast_ref::<hyper::Error>() {
            if hyper.is_incomplete_message() {
                return FetchError::Disconnected(error);
            }
            if hyper.is_parse() {
                return FetchError::Malformed(error);
            }
        }
        if let Some(io) = current.downcast_ref::<io::Error>() {
            socket = Some(io);
        }
        cause = current.source();
    }
    match socket {
        Some(io) => FetchError::Socket {
            reset: io.kind() == io::ErrorKind::ConnectionReset,
            errno: io.raw_os_error(),
            source: error,
        },
        None => FetchError::Other(error),
    }
}

fn body_error(error: reqwest::Error) -> FetchError {
    if error.is_timeout() {
        FetchError::BodyTimeout(error)
    } else {
        FetchError::BodyCut(error)
    }
}

async fn fetch_to(
    url: &str,
    path: &Path,
    buffer_size: usize,
    timeouts: Timeouts,
) -> Result<(), DownloadError> {
    let url = parse_url(url)?;
    // As the python's aiohttp, the environment's proxy is ignored.
    let client = reqwest::Client::builder()
        .no_proxy()
        .timeout(timeouts.total)
        .connect_timeout(timeouts.connect)
        .build()
        .map_err(FetchError::Other)?;
    let mut response = client.get(url).send().await.map_err(request_error)?;
    let status = response.status();
    if status != reqwest::StatusCode::OK {
        return Err(FetchError::Status(status).into());
    }

    let file = tokio::fs::File::create(path)
        .await
        .map_err(DownloadError::Write)?;
    let mut writer = BufWriter::with_capacity(buffer_size, file);
    while let Some(chunk) = response.chunk().await.map_err(body_error)? {
        for slice in chunk.chunks(buffer_size) {
            writer
                .write_all(slice)
                .await
                .map_err(DownloadError::Write)?;
        }
    }
    writer.flush().await.map_err(DownloadError::Write)?;
    Ok(())
}

fn archive_parent(dest: &Path) -> Option<&Path> {
    match dest.parent() {
        Some(parent) if parent.as_os_str().is_empty() => Some(Path::new(".")),
        parent => parent,
    }
}

fn extracted_dir(tmp_path: &Path) -> Result<PathBuf> {
    let mut dirs: Vec<PathBuf> = std::fs::read_dir(tmp_path)
        .with_context(|| format!("reading {}", tmp_path.display()))?
        .filter_map(std::result::Result::ok)
        .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_dir()))
        .map(|entry| entry.path())
        .collect();
    dirs.sort();

    if dirs.len() > 1 {
        warn!(
            count = dirs.len(),
            "multiple directories found in archive, using the first one"
        );
    }

    dirs.into_iter()
        .next()
        .context("no directory found in the extracted archive")
}

fn move_or_copy(from: &Path, to: &Path) -> Result<()> {
    if let Err(rename_error) = std::fs::rename(from, to) {
        super::fs::copy_dir_all(from, to).with_context(|| {
            format!(
                "moving {} to {} failed with {rename_error}, and so did copying it",
                from.display(),
                to.display()
            )
        })?;
    }
    Ok(())
}

// A `.git` file or link would redirect git to a repository outside the archive.
fn keep_only_history(repo: &Path) -> Result<()> {
    let git_dir = repo.join(".git");
    let is_dir = std::fs::symlink_metadata(&git_dir).is_ok_and(|meta| meta.is_dir());
    if !is_dir {
        bail!("no .git directory found in the extracted archive");
    }

    for entry in
        std::fs::read_dir(&git_dir).with_context(|| format!("reading {}", git_dir.display()))?
    {
        let entry = entry.with_context(|| format!("reading {}", git_dir.display()))?;
        if GIT_DIR_DATA.iter().any(|kept| entry.file_name() == *kept) {
            continue;
        }
        let path = entry.path();
        let removed = if entry.file_type().is_ok_and(|kind| kind.is_dir()) {
            std::fs::remove_dir_all(&path)
        } else {
            std::fs::remove_file(&path)
        };
        removed.with_context(|| format!("removing {}", path.display()))?;
    }
    for pointer in OBJECT_STORE_POINTERS {
        let path = git_dir.join("objects/info").join(pointer);
        match std::fs::remove_file(&path) {
            Err(error) if error.kind() != std::io::ErrorKind::NotFound => {
                return Err(error).with_context(|| format!("removing {}", path.display()));
            }
            _ => {}
        }
    }

    let config = git_dir.join("config");
    std::fs::write(&config, TRUSTED_CONFIG).with_context(|| format!("writing {}", config.display()))
}

async fn extract_and_replace(archive_path: &Path, tmp_path: &Path, dest: &Path) -> Result<()> {
    let archive = tokio::fs::File::open(archive_path)
        .await
        .with_context(|| format!("opening {}", archive_path.display()))?;
    super::tar::extract(archive, tmp_path).await?;
    let _ = tokio::fs::remove_file(archive_path).await;

    let (tmp_path, dest) = (tmp_path.to_path_buf(), dest.to_path_buf());
    tokio::task::spawn_blocking(move || replace_destination(&tmp_path, &dest)).await?
}

fn replace_destination(tmp_path: &Path, dest: &Path) -> Result<()> {
    let extracted = extracted_dir(tmp_path)?;
    // Still inside the temporary directory, so a failure leaves nothing behind.
    keep_only_history(&extracted)?;

    if dest.exists() {
        std::fs::remove_dir_all(dest).with_context(|| format!("removing {}", dest.display()))?;
    }

    move_or_copy(&extracted, dest)
}

async fn place_repo(url: &str, dest: &Path, buffer_size: Option<NonZeroUsize>) -> bool {
    let Some(parent) = archive_parent(dest) else {
        warn!("the destination has no parent directory");
        return false;
    };
    if let Err(e) = std::fs::create_dir_all(parent) {
        warn!(error = %e, "failed to create parent dir");
        return false;
    }

    let tmp_dir = match tempfile::Builder::new().prefix(TMP_PREFIX).tempdir() {
        Ok(d) => d,
        Err(e) => {
            warn!(error = %e, "failed to create temp dir");
            return false;
        }
    };
    let tmp_path = tmp_dir.path().to_path_buf();
    let archive_path = tmp_path.join(ARCHIVE_NAME);

    let buffer_size = resolve_buffer_size(buffer_size);
    if let Err(e) = fetch_to(url, &archive_path, buffer_size, TIMEOUTS).await {
        warn!(error = %e, "failed to download");
        return false;
    }

    if let Err(e) = extract_and_replace(&archive_path, &tmp_path, dest).await {
        warn!(error = %e, "failed to extract archive");
        return false;
    }

    true
}

/// Leaves the repository behind `url` at `dest`.
///
/// The archive holds only `.git`, so the working tree is restored with a
/// `git reset --hard`. Pass `checkout = false` to keep just the history: peak
/// disk stays at the size of the archive plus `.git`, which is what a consumer
/// that walks commits needs. `git_ignore` prunes working-tree files and has no
/// effect on a history-only download. `buffer_size` bounds a single write of
/// the archive to disk; `None` takes 64 KiB, and anything above 16 MiB is
/// taken as 16 MiB. Anything already at `dest` is replaced.
/// The archive is untrusted, so only its history is kept and its config is replaced.
pub async fn download_and_extract_repo(
    url: &str,
    dest: &Path,
    git_ignore: &[String],
    buffer_size: Option<NonZeroUsize>,
    checkout: bool,
) -> bool {
    if !place_repo(url, dest, buffer_size).await {
        return false;
    }

    if !checkout {
        return true;
    }

    if super::cli::reset_hard(dest).await.is_err() {
        let _ = std::fs::remove_dir_all(dest);
        return false;
    }

    let dest_path = dest.to_path_buf();
    let patterns = git_ignore.to_vec();
    let pruned = tokio::task::spawn_blocking(move || {
        super::fs::remove_symlinks(&dest_path)?;
        super::fs::delete_gitignored_files(&dest_path, &patterns)
    })
    .await;
    match pruned {
        Ok(Ok(())) => true,
        Ok(Err(error)) => {
            warn!(error = %error, "failed to prune the working tree");
            false
        }
        Err(error) => {
            warn!(error = %error, "failed to prune the working tree");
            false
        }
    }
}

#[cfg(test)]
mod tests {
    use super::{
        extract_and_replace, extracted_dir, fetch_to, resolve_buffer_size, Timeouts,
        DEFAULT_DOWNLOAD_BUFFER_SIZE, MAX_DOWNLOAD_BUFFER_SIZE, TRUSTED_CONFIG,
    };
    use crate::errors::{DownloadError, FetchError};
    use flate2::write::GzEncoder;
    use flate2::Compression;
    use std::num::NonZeroUsize;
    use std::path::{Path, PathBuf};
    use std::time::Duration;
    use tokio::io::{AsyncReadExt, AsyncWriteExt};
    use tokio::net::TcpListener;

    fn write_tar_gz(path: &Path, name: &str, contents: &[u8]) {
        let mut builder = tar::Builder::new(GzEncoder::new(Vec::new(), Compression::default()));
        let mut header = tar::Header::new_gnu();
        header.set_size(u64::try_from(contents.len()).unwrap());
        header.set_mode(0o644);
        builder.append_data(&mut header, name, contents).unwrap();
        let archive = builder.into_inner().unwrap().finish().unwrap();
        std::fs::write(path, archive).unwrap();
    }

    #[tokio::test]
    async fn the_archive_replaces_an_existing_destination_and_is_deleted() {
        let tmp = tempfile::tempdir().unwrap();
        let work = tmp.path().join("work");
        std::fs::create_dir_all(&work).unwrap();
        let archive_path = work.join("repo.tar.gz");
        write_tar_gz(&archive_path, "repo/.git/HEAD", b"ref: refs/heads/trunk\n");

        let dest = tmp.path().join("dest");
        std::fs::create_dir_all(&dest).unwrap();
        std::fs::write(dest.join("stale"), b"old").unwrap();

        extract_and_replace(&archive_path, &work, &dest)
            .await
            .unwrap();

        assert_eq!(
            std::fs::read_to_string(dest.join(".git/HEAD")).unwrap(),
            "ref: refs/heads/trunk\n"
        );
        assert!(!dest.join("stale").exists());
        assert!(!archive_path.exists());
    }

    #[tokio::test]
    async fn an_archive_without_a_directory_is_rejected() {
        let tmp = tempfile::tempdir().unwrap();
        let work = tmp.path().join("work");
        std::fs::create_dir_all(&work).unwrap();
        let archive_path = work.join("repo.tar.gz");
        write_tar_gz(&archive_path, "HEAD", b"ref: refs/heads/trunk\n");

        let dest = tmp.path().join("dest");

        assert!(extract_and_replace(&archive_path, &work, &dest)
            .await
            .is_err());
        assert!(!dest.exists());
    }

    const SHORT: Duration = Duration::from_secs(1);
    const LONG: Duration = Duration::from_secs(30);

    async fn answer_and_stall(mut stream: tokio::net::TcpStream, raw: &'static [u8]) {
        let mut request = [0u8; 1024];
        let _ = stream.read(&mut request).await;
        let _ = stream.write_all(raw).await;
        while stream.read(&mut request).await.is_ok_and(|read| read > 0) {}
    }

    // Answers every connection with `raw` and keeps it open, so only a
    // timeout of the client ends the request.
    async fn serve_and_stall(raw: &'static [u8]) -> String {
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let address = listener.local_addr().unwrap();
        tokio::spawn(async move {
            while let Ok((stream, _)) = listener.accept().await {
                tokio::spawn(answer_and_stall(stream, raw));
            }
        });
        format!("http://{address}/repo.tar.gz")
    }

    async fn fetch_with(url: &str, timeouts: Timeouts) -> Result<(), DownloadError> {
        let dir = tempfile::tempdir().unwrap();
        fetch_to(
            url,
            &dir.path().join("repo.tar.gz"),
            DEFAULT_DOWNLOAD_BUFFER_SIZE,
            timeouts,
        )
        .await
    }

    #[tokio::test]
    async fn a_connect_that_never_completes_is_a_connect_timeout() {
        // The connect covers the TLS handshake, which this server never answers.
        let url = serve_and_stall(b"")
            .await
            .replacen("http://", "https://", 1);

        let outcome = fetch_with(
            &url,
            Timeouts {
                connect: SHORT,
                total: LONG,
            },
        )
        .await;

        assert!(
            matches!(
                outcome,
                Err(DownloadError::Fetch(FetchError::ConnectTimeout(_)))
            ),
            "{outcome:?}"
        );
    }

    #[tokio::test]
    async fn a_response_that_never_starts_is_a_timeout_before_the_headers() {
        let url = serve_and_stall(b"").await;

        let outcome = fetch_with(
            &url,
            Timeouts {
                connect: LONG,
                total: SHORT,
            },
        )
        .await;

        assert!(
            matches!(outcome, Err(DownloadError::Fetch(FetchError::Timeout(_)))),
            "{outcome:?}"
        );
    }

    #[tokio::test]
    async fn a_body_that_stops_arriving_is_a_timeout_reading_the_body() {
        let url =
            serve_and_stall(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n\x1f\x8b\x08\x00")
                .await;

        let outcome = fetch_with(
            &url,
            Timeouts {
                connect: LONG,
                total: SHORT,
            },
        )
        .await;

        assert!(
            matches!(
                outcome,
                Err(DownloadError::Fetch(FetchError::BodyTimeout(_)))
            ),
            "{outcome:?}"
        );
    }

    #[test]
    fn an_absent_buffer_size_takes_the_default_of_the_python() {
        assert_eq!(resolve_buffer_size(None), DEFAULT_DOWNLOAD_BUFFER_SIZE);
        assert_eq!(resolve_buffer_size(NonZeroUsize::new(8 * 1024)), 8 * 1024);
    }

    #[test]
    fn a_buffer_size_above_the_maximum_is_clamped() {
        assert_eq!(
            resolve_buffer_size(NonZeroUsize::new(MAX_DOWNLOAD_BUFFER_SIZE + 1)),
            MAX_DOWNLOAD_BUFFER_SIZE
        );
    }

    fn git(repo: &Path, args: &[&str]) {
        let status = std::process::Command::new("git")
            .arg("-C")
            .arg(repo)
            .args([
                "-c",
                "user.email=dev@fluidattacks.com",
                "-c",
                "user.name=dev",
            ])
            .args(args)
            .status()
            .unwrap();
        assert!(status.success());
    }

    fn hostile_archive(dir: &Path, key: &str) -> (PathBuf, PathBuf) {
        let marker = dir.join("pwned");
        let payload = dir.join("payload.sh");
        std::fs::write(
            &payload,
            format!("#!/bin/sh\ntouch {}\ncat\n", marker.display()),
        )
        .unwrap();
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt;
            std::fs::set_permissions(&payload, std::fs::Permissions::from_mode(0o755)).unwrap();
        }

        let source = dir.join("source");
        std::fs::create_dir_all(&source).unwrap();
        git(&source, &["init", "-q"]);
        std::fs::write(source.join(".gitattributes"), b"* filter=pwn\n").unwrap();
        std::fs::write(source.join("tracked.txt"), b"content\n").unwrap();
        git(&source, &["add", "-A"]);
        git(&source, &["commit", "-q", "-m", "add tracked file"]);
        git(&source, &["config", key, &payload.to_string_lossy()]);
        std::fs::create_dir_all(source.join(".git/objects/info")).unwrap();
        std::fs::write(
            source.join(".git/objects/info/alternates"),
            format!("{}\n", dir.display()),
        )
        .unwrap();

        let archive = dir.join("work/repo.tar.gz");
        std::fs::create_dir_all(archive.parent().unwrap()).unwrap();
        let mut builder = tar::Builder::new(GzEncoder::new(Vec::new(), Compression::default()));
        builder
            .append_dir_all("source/.git", source.join(".git"))
            .unwrap();
        std::fs::write(&archive, builder.into_inner().unwrap().finish().unwrap()).unwrap();
        std::fs::remove_dir_all(&source).unwrap();
        (archive, marker)
    }

    #[tokio::test]
    async fn a_restored_archive_runs_nothing_it_configures() {
        for key in ["filter.pwn.smudge", "filter.pwn.process", "core.fsmonitor"] {
            let tmp = tempfile::tempdir().unwrap();
            let (archive, marker) = hostile_archive(tmp.path(), key);
            let dest = tmp.path().join("dest");

            extract_and_replace(&archive, archive.parent().unwrap(), &dest)
                .await
                .unwrap();
            crate::cli::reset_hard(&dest).await.unwrap();

            assert!(!marker.exists(), "{key} ran");
            assert_eq!(
                std::fs::read_to_string(dest.join("tracked.txt")).unwrap(),
                "content\n"
            );
        }
    }

    #[tokio::test]
    async fn only_the_history_of_the_archive_is_kept() {
        let tmp = tempfile::tempdir().unwrap();
        let (archive, _) = hostile_archive(tmp.path(), "core.fsmonitor");
        let dest = tmp.path().join("dest");

        extract_and_replace(&archive, archive.parent().unwrap(), &dest)
            .await
            .unwrap();

        let git_dir = dest.join(".git");
        assert_eq!(
            std::fs::read_to_string(git_dir.join("config")).unwrap(),
            TRUSTED_CONFIG
        );
        assert!(!git_dir.join("hooks").exists());
        assert!(!git_dir.join("objects/info/alternates").exists());
        assert!(git_dir.join("HEAD").is_file());
    }

    #[tokio::test]
    async fn a_git_file_in_the_archive_is_rejected() {
        let tmp = tempfile::tempdir().unwrap();
        let work = tmp.path().join("work");
        std::fs::create_dir_all(&work).unwrap();
        let archive_path = work.join("repo.tar.gz");
        write_tar_gz(&archive_path, "repo/.git", b"gitdir: /elsewhere/.git\n");
        let dest = tmp.path().join("dest");

        assert!(extract_and_replace(&archive_path, &work, &dest)
            .await
            .is_err());
        assert!(!dest.exists());
    }

    #[test]
    fn the_first_directory_is_taken_in_a_stable_order() {
        let tmp = tempfile::tempdir().unwrap();
        std::fs::write(tmp.path().join("repo.tar.gz"), b"").unwrap();
        std::fs::create_dir_all(tmp.path().join("b_repo")).unwrap();
        std::fs::create_dir_all(tmp.path().join("a_repo")).unwrap();

        assert_eq!(
            extracted_dir(tmp.path()).unwrap(),
            tmp.path().join("a_repo")
        );
    }
}