mktool 1.5.7

/*
 * Copyright (c) 2026 Jonathan Perkin <jonathan@perkin.org.uk>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

use std::fs;
use std::io::Write;
use std::net::TcpListener;
use std::path::Path;
use std::process::{Child, Command, Stdio};
use std::thread;
use std::time::{Duration, Instant};

const MKTOOL: &str = env!("CARGO_BIN_EXE_mktool");

type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;

fn free_port() -> Result<u16> {
    Ok(TcpListener::bind("127.0.0.1:0")?.local_addr()?.port())
}

fn start_nc(port: u16, initial_response: &str) -> Result<Child> {
    let child = Command::new("sh")
        .args([
            "-c",
            &format!(
                "(printf '{initial_response}'; cat) | nc -l 127.0.0.1 {port}"
            ),
        ])
        .stdin(Stdio::null())
        .stdout(Stdio::null())
        .stderr(Stdio::null())
        .spawn()?;
    wait_for_port(port);
    Ok(child)
}

fn wait_for_port(port: u16) {
    let deadline = Instant::now() + Duration::from_secs(5);
    while Instant::now() < deadline {
        if TcpListener::bind(("127.0.0.1", port)).is_err() {
            return;
        }
        thread::sleep(Duration::from_millis(10));
    }
    panic!("nc did not start listening on port {port} in time");
}

fn has_temp_files(dir: &Path) -> Result<bool> {
    for entry in fs::read_dir(dir)? {
        if entry?.file_name().to_string_lossy().contains(".mktool.") {
            return Ok(true);
        }
    }
    Ok(false)
}

/*
 * Verify that fetching from an HTTPS server that negotiates HTTP/2 via
 * ALPN does not panic.  This catches the case where the http2 feature is
 * missing from the reqwest dependency, which causes hyper-util to panic
 * at runtime when the TLS layer negotiates h2.
 *
 * Uses the non-"-" site prefix format so that the filename is appended
 * to the site URL, exercising the url_from_site() non-direct path.
 */
#[test]
fn fetch_https_http2() -> Result<()> {
    let dir = tempfile::tempdir()?;
    let distdir = dir.path().to_str().ok_or("invalid tempdir path")?;

    let input = format!("robots.txt {distdir} https://www.google.com\n");

    let mut child = Command::new(MKTOOL)
        .args(["fetch", "-d", distdir, "-I", "-"])
        .env("MKTOOL_JOBS", "1")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()?;

    let mut stdin = child.stdin.take().ok_or("failed to open stdin")?;
    stdin.write_all(input.as_bytes())?;
    drop(stdin);

    let output = child.wait_with_output()?;
    let stderr = String::from_utf8_lossy(&output.stderr);

    assert!(output.status.success(), "fetch failed: {stderr}");
    assert!(
        dir.path().join("robots.txt").exists(),
        "downloaded file not found"
    );
    assert!(!has_temp_files(dir.path())?, "temp file not cleaned up");
    Ok(())
}

/*
 * Verify FTP fetch works, reading input from a file rather than stdin.
 * Uses a subdirectory in the filepath to exercise create_dir_all().
 */
#[test]
fn fetch_ftp() -> Result<()> {
    let dir = tempfile::tempdir()?;
    let distdir = dir.path().to_str().ok_or("invalid tempdir path")?;

    let input_file = dir.path().join("input");
    fs::write(
        &input_file,
        format!("sub/robots.txt {distdir} -ftp://ftp.netbsd.org/robots.txt\n"),
    )?;

    let output = Command::new(MKTOOL)
        .args([
            "fetch",
            "-d",
            distdir,
            "-I",
            input_file.to_str().ok_or("invalid input path")?,
        ])
        .env("MKTOOL_JOBS", "1")
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .output()?;

    let stderr = String::from_utf8_lossy(&output.stderr);

    assert!(output.status.success(), "fetch failed: {stderr}");
    assert!(
        dir.path().join("sub").join("robots.txt").exists(),
        "downloaded file not found"
    );
    assert!(!has_temp_files(dir.path())?, "temp file not cleaned up");
    Ok(())
}

/*
 * Verify that a checksum mismatch causes the fetch to fail and that no
 * partial or incorrect file is left behind.
 */
#[test]
fn fetch_https_bad_checksum() -> Result<()> {
    let dir = tempfile::tempdir()?;
    let distdir = dir.path().to_str().ok_or("invalid tempdir path")?;

    let distinfo = dir.path().join("distinfo");
    fs::write(
        &distinfo,
        "BLAKE2s (robots.txt) = 0000000000000000000000000000000000000000000000000000000000000000\n",
    )?;

    let input =
        format!("robots.txt {distdir} -https://www.google.com/robots.txt\n");

    let mut child = Command::new(MKTOOL)
        .args([
            "fetch",
            "-d",
            distdir,
            "-f",
            distinfo.to_str().ok_or("invalid distinfo path")?,
            "-I",
            "-",
        ])
        .env("MKTOOL_JOBS", "1")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()?;

    let mut stdin = child.stdin.take().ok_or("failed to open stdin")?;
    stdin.write_all(input.as_bytes())?;
    drop(stdin);

    let output = child.wait_with_output()?;
    let stderr = String::from_utf8_lossy(&output.stderr);

    assert!(!output.status.success(), "fetch should have failed: {stderr}");
    assert!(
        stderr.contains("Verification failed"),
        "expected checksum verification failure: {stderr}"
    );
    assert!(
        !dir.path().join("robots.txt").exists(),
        "failed file should have been cleaned up"
    );
    assert!(!has_temp_files(dir.path())?, "temp file not cleaned up");
    Ok(())
}

#[test]
fn fetch_ftp_bad_checksum() -> Result<()> {
    let dir = tempfile::tempdir()?;
    let distdir = dir.path().to_str().ok_or("invalid tempdir path")?;

    let distinfo = dir.path().join("distinfo");
    fs::write(
        &distinfo,
        "BLAKE2s (robots.txt) = 0000000000000000000000000000000000000000000000000000000000000000\n",
    )?;

    let input =
        format!("robots.txt {distdir} -ftp://ftp.netbsd.org/robots.txt\n");

    let mut child = Command::new(MKTOOL)
        .args([
            "fetch",
            "-d",
            distdir,
            "-f",
            distinfo.to_str().ok_or("invalid distinfo path")?,
            "-I",
            "-",
        ])
        .env("MKTOOL_JOBS", "1")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()?;

    let mut stdin = child.stdin.take().ok_or("failed to open stdin")?;
    stdin.write_all(input.as_bytes())?;
    drop(stdin);

    let output = child.wait_with_output()?;
    let stderr = String::from_utf8_lossy(&output.stderr);

    assert!(!output.status.success(), "fetch should have failed: {stderr}");
    assert!(
        stderr.contains("Verification failed"),
        "expected checksum verification failure: {stderr}"
    );
    assert!(
        !dir.path().join("robots.txt").exists(),
        "failed file should have been cleaned up"
    );
    assert!(!has_temp_files(dir.path())?, "temp file not cleaned up");
    Ok(())
}

/*
 * Verify that a stalled FTP server triggers a read timeout and does not
 * hang indefinitely.  The nc listener sends the FTP greeting but never
 * responds to any commands.
 */
#[test]
fn fetch_ftp_read_timeout() -> Result<()> {
    let port = free_port()?;
    let mut nc = start_nc(port, "220 ready\\r\\n")?;

    let dir = tempfile::tempdir()?;
    let distdir = dir.path().to_str().ok_or("invalid tempdir path")?;
    let input =
        format!("test.txt {distdir} -ftp://127.0.0.1:{port}/test.txt\n");

    let start = Instant::now();

    let mut child = Command::new(MKTOOL)
        .args(["fetch", "-d", distdir, "-I", "-"])
        .env("MKTOOL_JOBS", "1")
        .env("MKTOOL_READ_TIMEOUT", "2")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()?;

    let mut stdin = child.stdin.take().ok_or("failed to open stdin")?;
    stdin.write_all(input.as_bytes())?;
    drop(stdin);

    let output = child.wait_with_output()?;
    let elapsed = start.elapsed();

    let _ = nc.kill();
    let _ = nc.wait();

    let stderr = String::from_utf8_lossy(&output.stderr);

    assert!(!output.status.success(), "fetch should have failed: {stderr}");
    assert!(
        elapsed.as_secs() < 30,
        "read timeout did not fire, took {elapsed:?}"
    );
    assert!(!has_temp_files(dir.path())?, "temp file not cleaned up");
    Ok(())
}

/*
 * Verify that invalid input (too few fields) is handled gracefully.
 */
#[test]
fn fetch_invalid_input() -> Result<()> {
    let dir = tempfile::tempdir()?;
    let distdir = dir.path().to_str().ok_or("invalid tempdir path")?;

    let mut child = Command::new(MKTOOL)
        .args(["fetch", "-d", distdir, "-j", "1", "-I", "-"])
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()?;

    let mut stdin = child.stdin.take().ok_or("failed to open stdin")?;
    stdin.write_all(b"onlyonefield\n")?;
    drop(stdin);

    let output = child.wait_with_output()?;
    let stderr = String::from_utf8_lossy(&output.stderr);

    assert!(!output.status.success(), "fetch should have failed: {stderr}");
    assert!(
        stderr.contains("Invalid input"),
        "expected invalid input error: {stderr}"
    );
    Ok(())
}

/*
 * Verify that an HTTP 404 response is handled gracefully.
 */
#[test]
fn fetch_http_404() -> Result<()> {
    let port = free_port()?;
    let mut nc = start_nc(
        port,
        "HTTP/1.1 404 Not Found\\r\\nContent-Length: 0\\r\\n\\r\\n",
    )?;

    let dir = tempfile::tempdir()?;
    let distdir = dir.path().to_str().ok_or("invalid tempdir path")?;
    let input =
        format!("test.txt {distdir} -http://127.0.0.1:{port}/test.txt\n");

    let mut child = Command::new(MKTOOL)
        .args(["fetch", "-d", distdir, "-I", "-"])
        .env("MKTOOL_JOBS", "1")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()?;

    let mut stdin = child.stdin.take().ok_or("failed to open stdin")?;
    stdin.write_all(input.as_bytes())?;
    drop(stdin);

    let output = child.wait_with_output()?;

    let _ = nc.kill();
    let _ = nc.wait();

    let stderr = String::from_utf8_lossy(&output.stderr);

    assert!(!output.status.success(), "fetch should have failed: {stderr}");
    assert!(stderr.contains("404"), "expected 404 error in output: {stderr}");
    assert!(
        !dir.path().join("test.txt").exists(),
        "no file should have been created"
    );
    assert!(!has_temp_files(dir.path())?, "temp file not cleaned up");
    Ok(())
}

/*
 * Verify that re-fetching an already-downloaded file is a no-op.
 */
#[test]
fn fetch_https_refetch() -> Result<()> {
    let dir = tempfile::tempdir()?;
    let distdir = dir.path().to_str().ok_or("invalid tempdir path")?;

    let input =
        format!("robots.txt {distdir} -https://www.google.com/robots.txt\n");

    for _ in 0..2 {
        let mut child = Command::new(MKTOOL)
            .args(["fetch", "-d", distdir, "-I", "-"])
            .env("MKTOOL_JOBS", "1")
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .spawn()?;

        let mut stdin = child.stdin.take().ok_or("failed to open stdin")?;
        stdin.write_all(input.as_bytes())?;
        drop(stdin);

        let output = child.wait_with_output()?;
        let stderr = String::from_utf8_lossy(&output.stderr);

        assert!(output.status.success(), "fetch failed: {stderr}");
    }

    assert!(
        dir.path().join("robots.txt").exists(),
        "downloaded file not found"
    );
    assert!(!has_temp_files(dir.path())?, "temp file not cleaned up");
    Ok(())
}

/*
 * Verify that an HTTP connection error (nothing listening) is handled
 * gracefully.
 */
#[test]
fn fetch_http_connect_error() -> Result<()> {
    let port = free_port()?;

    let dir = tempfile::tempdir()?;
    let distdir = dir.path().to_str().ok_or("invalid tempdir path")?;
    let input =
        format!("test.txt {distdir} -http://127.0.0.1:{port}/test.txt\n");

    let mut child = Command::new(MKTOOL)
        .args(["fetch", "-d", distdir, "-I", "-"])
        .env("MKTOOL_JOBS", "1")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()?;

    let mut stdin = child.stdin.take().ok_or("failed to open stdin")?;
    stdin.write_all(input.as_bytes())?;
    drop(stdin);

    let output = child.wait_with_output()?;
    let stderr = String::from_utf8_lossy(&output.stderr);

    assert!(!output.status.success(), "fetch should have failed: {stderr}");
    assert!(
        stderr.contains("Unable to fetch"),
        "expected connection error: {stderr}"
    );
    assert!(
        !dir.path().join("test.txt").exists(),
        "no file should have been created"
    );
    assert!(!has_temp_files(dir.path())?, "temp file not cleaned up");
    Ok(())
}