agent-first-http 0.8.0

//! Integration tests for the `--render none` (HTTP fast path) fetch.

#![allow(
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::panic,
    clippy::disallowed_methods,
    clippy::disallowed_macros,
    clippy::err_expect,
    clippy::print_stdout,
    clippy::useless_conversion
)]

mod support;

use std::time::Duration;

use agent_first_http::sdk::Client;
use agent_first_http::sdk::fetch::RenderMode;
use agent_first_http::sdk::fetch::result::TraceStageStatus;
use agent_first_http::shared::artifacts::Artifact;
use agent_first_http::shared::error::ErrorCode;

#[tokio::test]
async fn http_only_fetch_writes_body_artifact() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let result = client
        .fetch(format!("{}/plain.html", fixture.base_url()))
        .render(RenderMode::None)
        .want([Artifact::Body])
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .expect("fetch");

    assert_eq!(result.status, 200);
    assert!(
        result.final_url.ends_with("/plain.html"),
        "final_url = {}",
        result.final_url
    );
    assert!(
        result.warnings.is_empty(),
        "warnings: {:?}",
        result.warnings
    );

    let body_file = result.body_file.as_ref().expect("body_file in response");
    assert!(body_file.exists(), "{} should exist", body_file.display());
    assert_eq!(
        body_file.extension().and_then(|s| s.to_str()),
        Some("html"),
        "body should be saved as .html for text/html content-type",
    );
    let bytes = tokio::fs::read(body_file).await.expect("read");
    let text = String::from_utf8(bytes).expect("utf8");
    assert!(
        text.contains("Hello"),
        "body content was not preserved: {text:?}"
    );

    assert_eq!(
        result.trace.render_decision,
        agent_first_http::sdk::fetch::result::RenderDecision::HttpOnly
    );
    assert!(result.trace.main_request_observed);
    // The new convenience fields agents use for retry logic.
    assert!(
        !result.trace.render_used,
        "HTTP-only path must report render_used=false"
    );
    assert_eq!(
        result.trace.render_mode,
        agent_first_http::sdk::fetch::result::TraceRenderMode::None,
        "render_mode should echo the agent's --render none request",
    );
    assert_eq!(result.trace.current_stage, "complete");
    assert_eq!(result.trace.timeout_ms, 30_000);
    assert!(
        result
            .trace
            .stages
            .iter()
            .any(|stage| { stage.name == "navigate" && stage.status == TraceStageStatus::Ok }),
        "success trace should include navigate stage: {:?}",
        result.trace.stages
    );
}

#[tokio::test]
async fn render_auto_default_want_keeps_plain_html_on_http_path() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let result = client
        .fetch(format!("{}/plain.html", fixture.base_url()))
        .render(RenderMode::Auto)
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .expect("fetch");

    assert_eq!(
        result.trace.render_decision,
        agent_first_http::sdk::fetch::result::RenderDecision::HttpOnly
    );
    assert!(!result.trace.render_used);
    assert!(result.body_file.is_some(), "body_file should be default");
    assert!(
        result.content_file.is_none(),
        "content_file should not force browser escalation by default"
    );
    assert!(result.tab_id.is_none(), "HTTP path should not open a tab");
}

#[tokio::test]
async fn http_only_flags_cloudflare_turnstile_as_bot_wall() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let result = client
        .fetch(format!("{}/cloudflare-turnstile.html", fixture.base_url()))
        .render(RenderMode::None)
        .want([Artifact::Body])
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .expect("fetch");

    assert_eq!(
        result.page_kind,
        Some(agent_first_http::sdk::fetch::PageKind::BotWallDetected)
    );
    assert!(
        result
            .warnings
            .iter()
            .any(|w| w.code == ErrorCode::BotWallDetected && w.detail.contains("human takeover")),
        "expected bot-wall warning with takeover guidance; got {:?}",
        result.warnings
    );
    let next = result.next_action.as_ref().expect("next_action");
    assert_eq!(
        next.kind,
        agent_first_http::sdk::fetch::result::NextActionKind::HumanTakeover
    );
    assert_eq!(
        next.recommended_command,
        format!(
            "afhttp fetch {}/cloudflare-turnstile.html --takeover",
            fixture.base_url()
        )
    );
    assert!(!next.target_content_verified);
}

#[tokio::test]
async fn http_only_max_response_bytes_truncates_with_warning() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");
    let client = Client::connect("ws://localhost:9999").expect("client");

    // /large-body returns 128 KiB; cap at 32 KiB and expect a truncation
    // warning plus exactly the prefix on disk.
    let result = client
        .fetch(format!("{}/large-body", fixture.base_url()))
        .render(RenderMode::None)
        .max_response_bytes(32 * 1024)
        .want([Artifact::Body])
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .expect("fetch");

    assert_eq!(result.status, 200);
    let body_file = result.body_file.as_ref().expect("body_file");
    let on_disk = tokio::fs::read(body_file).await.expect("read");
    assert_eq!(
        on_disk.len(),
        32 * 1024,
        "truncated body must be exactly the cap size",
    );
    assert!(
        result
            .warnings
            .iter()
            .any(|w| w.code == ErrorCode::NetworkBodyTruncated),
        "expected network_body_truncated warning; got {:?}",
        result.warnings
    );
}

#[tokio::test]
async fn http_only_max_response_bytes_zero_disables_cap() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");
    let client = Client::connect("ws://localhost:9999").expect("client");

    let result = client
        .fetch(format!("{}/large-body", fixture.base_url()))
        .render(RenderMode::None)
        .max_response_bytes(0)
        .want([Artifact::Body])
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .expect("fetch");

    let body_file = result.body_file.as_ref().expect("body_file");
    let on_disk = tokio::fs::read(body_file).await.expect("read");
    assert_eq!(on_disk.len(), 128 * 1024, "cap=0 must store the full body");
    assert!(
        result
            .warnings
            .iter()
            .all(|w| w.code != ErrorCode::NetworkBodyTruncated),
        "cap=0 must not produce a truncation warning",
    );
}

#[tokio::test]
async fn http_only_fetch_chooses_json_extension() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let result = client
        .fetch(format!("{}/data.json", fixture.base_url()))
        .render(RenderMode::None)
        .want([Artifact::Body])
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .expect("fetch");

    let body_file = result.body_file.as_ref().expect("body_file");
    assert_eq!(body_file.extension().and_then(|s| s.to_str()), Some("json"),);
}

#[tokio::test]
async fn http_only_fetch_applies_request_overrides() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let result = client
        .fetch(format!("{}/headers.json", fixture.base_url()))
        .render(RenderMode::None)
        .want([Artifact::Body])
        .header("X-Afhttp-Test", "present")
        .user_agent("afhttp-test-agent/1")
        .cookie("sid", "abc")
        .cookie("theme", "light")
        .cookie_full(
            cookie::Cookie::build(("scoped", "yes"))
                .path("/headers")
                .http_only(true)
                .same_site(cookie::SameSite::Lax)
                .build(),
        )
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .expect("fetch");

    let body_file = result.body_file.as_ref().expect("body_file");
    let body = tokio::fs::read_to_string(body_file).await.expect("read");
    let json: serde_json::Value = serde_json::from_str(&body).expect("json");
    assert_eq!(json["x-afhttp-test"], "present");
    assert_eq!(json["user-agent"], "afhttp-test-agent/1");
    assert_eq!(json["cookie"], "sid=abc; theme=light; scoped=yes");
}

#[tokio::test]
async fn header_user_agent_normalizes_to_user_agent_override() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let result = client
        .fetch(format!("{}/headers.json", fixture.base_url()))
        .render(RenderMode::None)
        .want([Artifact::Body])
        .header("User-Agent", "header-agent/1")
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .expect("fetch");

    let body_file = result.body_file.as_ref().expect("body_file");
    let body = tokio::fs::read_to_string(body_file).await.expect("read");
    let json: serde_json::Value = serde_json::from_str(&body).expect("json");
    assert_eq!(json["user-agent"], "header-agent/1");
}

#[tokio::test]
async fn user_agent_header_conflict_is_invalid_argument() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let err = client
        .fetch(format!("{}/headers.json", fixture.base_url()))
        .render(RenderMode::None)
        .header("User-Agent", "header-agent/1")
        .user_agent("method-agent/1")
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .err()
        .expect("error");
    assert_eq!(err.error_code, ErrorCode::InvalidArgument);
}

#[tokio::test]
async fn cookie_header_conflict_is_invalid_argument() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let err = client
        .fetch(format!("{}/headers.json", fixture.base_url()))
        .render(RenderMode::None)
        .header("Cookie", "raw=1")
        .cookie("sid", "abc")
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .err()
        .expect("error");
    assert_eq!(err.error_code, ErrorCode::InvalidArgument);
}

#[tokio::test]
async fn full_cookie_path_mismatch_is_invalid_argument() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let err = client
        .fetch(format!("{}/headers.json", fixture.base_url()))
        .render(RenderMode::None)
        .cookie_full(
            cookie::Cookie::build(("scoped", "yes"))
                .path("/other")
                .build(),
        )
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .err()
        .expect("error");
    assert_eq!(err.error_code, ErrorCode::InvalidArgument);
}

#[tokio::test]
async fn secure_cookie_on_http_url_is_skipped_not_invalid() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let result = client
        .fetch(format!("{}/headers.json", fixture.base_url()))
        .render(RenderMode::None)
        .want([Artifact::Body])
        .cookie_full(
            cookie::Cookie::build(("secure_only", "secret"))
                .secure(true)
                .build(),
        )
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .expect("fetch");

    let body_file = result.body_file.as_ref().expect("body_file");
    let body = tokio::fs::read_to_string(body_file).await.expect("read");
    let json: serde_json::Value = serde_json::from_str(&body).expect("json");
    assert!(json["cookie"].is_null(), "secure cookie leaked over HTTP");
}

#[tokio::test]
async fn evaluate_after_wait_requires_browser_path() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let err = client
        .fetch(format!("{}/plain.html", fixture.base_url()))
        .render(RenderMode::None)
        .evaluate_after_wait("document.body.dataset.x = '1'")
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .err()
        .expect("error");
    assert_eq!(err.error_code, ErrorCode::InvalidArgument);
}

#[tokio::test]
async fn http_only_fetch_follows_redirect() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let result = client
        .fetch(format!("{}/redirect", fixture.base_url()))
        .render(RenderMode::None)
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .expect("fetch");

    assert_eq!(result.status, 200);
    assert!(
        result.final_url.ends_with("/plain.html"),
        "redirect should land on /plain.html, got {}",
        result.final_url,
    );
}

#[tokio::test]
async fn http_only_fetch_records_404_status() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");

    let client = Client::connect("ws://localhost:9999").expect("client");
    let result = client
        .fetch(format!("{}/404", fixture.base_url()))
        .render(RenderMode::None)
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .expect("fetch");

    assert_eq!(
        result.status, 404,
        "404 is not an error; it's a structured fact"
    );
}

#[tokio::test]
async fn http_only_fetch_on_unreachable_host_returns_target_unreachable() {
    let tmpdir = tempfile::tempdir().expect("tempdir");
    let client = Client::connect("ws://localhost:9999").expect("client");
    let err = client
        .fetch("http://127.0.0.1:1") // port 1 is reserved; connect fails fast
        .render(RenderMode::None)
        .out_dir(tmpdir.path().to_path_buf())
        .send()
        .await
        .err()
        .expect("expected an error");
    assert_eq!(
        err.error_code,
        agent_first_http::shared::error::ErrorCode::TargetUnreachable,
        "got {err:?}"
    );
    assert!(err.retryable, "target_unreachable should be retryable");
}

#[tokio::test]
async fn detailed_fetch_error_preserves_timeout_trace() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");
    let client = Client::connect("ws://localhost:9999").expect("client");

    // Disable the cookie jar so setup does not make a `GET /profile` call to the
    // dead endpoint above: that connection-refused is near-instant on Unix but
    // slow on Windows, where it would otherwise eat a tight timeout budget
    // before `navigate` is ever reached. With the jar off, setup is instant and
    // the deadline trips deterministically in `navigate` waiting on slow.html
    // (which sleeps 500ms server-side) on every platform.
    let err = client
        .fetch(format!("{}/slow.html", fixture.base_url()))
        .render(RenderMode::None)
        .no_cookie_jar()
        .timeout(Duration::from_millis(200))
        .want([Artifact::Body])
        .out_dir(tmpdir.path().to_path_buf())
        .send_detailed()
        .await
        .err()
        .expect("expected timeout");

    assert_eq!(err.error_code, ErrorCode::NavigationTimeout);
    assert_eq!(err.trace.timeout_ms, 200);
    assert_eq!(err.trace.current_stage, "navigate");
    assert!(
        err.trace
            .stages
            .iter()
            .any(|stage| { stage.name == "navigate" && stage.status == TraceStageStatus::Timeout }),
        "timeout trace should mark navigate timeout: {:?}",
        err.trace.stages
    );
}

#[tokio::test]
async fn cli_render_none_without_endpoint_does_not_require_browser_env() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");
    let output = tokio::process::Command::new(env!("CARGO_BIN_EXE_afhttp"))
        .arg("fetch")
        .arg(format!("{}/plain.html", fixture.base_url()))
        .arg("--render")
        .arg("none")
        .arg("--want")
        .arg("body")
        .arg("--timeout-ms")
        .arg("5000")
        .arg("--out")
        .arg(tmpdir.path())
        .env_remove("AFHTTP_TEST_BROWSER_BIN")
        .output()
        .await
        .expect("run afhttp");
    assert!(
        output.status.success(),
        "afhttp failed: stdout={} stderr={}",
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr)
    );
    let body: serde_json::Value = serde_json::from_slice(&output.stdout).expect("json");
    assert_eq!(body["kind"], "result");
    agent_first_data::validate_protocol_event(&body, true).expect("protocol event");
    let body = &body["result"];
    assert_eq!(body["code"], "fetch");
    assert_eq!(body["trace"]["render_decision"], "http_only");
    assert_eq!(body["trace"]["timeout_ms"], 5000);
    assert!(
        body["trace"]["stages"]
            .as_array()
            .is_some_and(|s| !s.is_empty())
    );
    assert!(body.get("artifacts").is_none(), "fetch output must be flat");
    let body_file = body["body_file"].as_str().expect("body_file");
    assert!(
        std::path::Path::new(body_file).is_absolute(),
        "body_file must be absolute: {body_file}"
    );
}

#[tokio::test]
async fn cli_render_auto_without_want_does_not_require_browser_for_plain_html() {
    let fixture = support::fixture_server::spawn().await;
    let tmpdir = tempfile::tempdir().expect("tempdir");
    let output = tokio::process::Command::new(env!("CARGO_BIN_EXE_afhttp"))
        .arg("fetch")
        .arg(format!("{}/plain.html", fixture.base_url()))
        .arg("--render")
        .arg("auto")
        .arg("--timeout-ms")
        .arg("5000")
        .arg("--out")
        .arg(tmpdir.path())
        .env_remove("AFHTTP_TEST_BROWSER_BIN")
        .output()
        .await
        .expect("run afhttp");
    assert!(
        output.status.success(),
        "afhttp failed: stdout={} stderr={}",
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr)
    );
    let body: serde_json::Value = serde_json::from_slice(&output.stdout).expect("json");
    assert_eq!(body["kind"], "result");
    agent_first_data::validate_protocol_event(&body, true).expect("protocol event");
    let body = &body["result"];
    assert_eq!(body["code"], "fetch");
    assert_eq!(body["trace"]["render_decision"], "http_only");
    assert_eq!(body["trace"]["render_used"], false);
    assert!(body["body_file"].as_str().is_some());
    assert!(
        body.get("content_file").is_none(),
        "implicit --want must not include browser-only content artifacts"
    );
    assert!(
        body.get("tab_id").is_none(),
        "plain HTML should not start the lazy inline browser"
    );
}

#[tokio::test]
async fn cli_fetch_error_envelope_includes_trace() {
    let tmpdir = tempfile::tempdir().expect("tempdir");
    let output = tokio::process::Command::new(env!("CARGO_BIN_EXE_afhttp"))
        .arg("fetch")
        .arg("http://127.0.0.1:1/will-fail")
        .arg("--render")
        .arg("none")
        .arg("--want")
        .arg("body")
        .arg("--timeout-ms")
        .arg("1000")
        .arg("--out")
        .arg(tmpdir.path())
        .output()
        .await
        .expect("run afhttp");
    assert!(
        !output.status.success(),
        "fetch unexpectedly succeeded: stdout={} stderr={}",
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr)
    );
    let body: serde_json::Value = serde_json::from_slice(&output.stdout).expect("json");
    assert_eq!(body["kind"], "error");
    agent_first_data::validate_protocol_event(&body, true).expect("protocol event");
    assert!(
        body["trace"].is_object(),
        "error body missing trace: {body}"
    );
    assert_eq!(body["trace"]["timeout_ms"], 1000);
    assert_eq!(body["trace"]["render_mode"], "none");
    assert!(
        body["trace"]["stages"]
            .as_array()
            .is_some_and(|s| !s.is_empty())
    );
}

#[tokio::test]
async fn cli_fetch_rejects_legacy_timeout_flag() {
    let fixture = support::fixture_server::spawn().await;
    let output = tokio::process::Command::new(env!("CARGO_BIN_EXE_afhttp"))
        .arg("fetch")
        .arg(format!("{}/plain.html", fixture.base_url()))
        .arg("--render")
        .arg("none")
        .arg("--timeout")
        .arg("1s")
        .output()
        .await
        .expect("run afhttp");
    assert!(
        !output.status.success(),
        "legacy --timeout unexpectedly succeeded: stdout={} stderr={}",
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr)
    );
    let body: serde_json::Value = serde_json::from_slice(&output.stdout).expect("json");
    assert_eq!(body["kind"], "error");
    agent_first_data::validate_protocol_event(&body, true).expect("protocol event");
    assert_eq!(body["error"]["code"], "invalid_argument");
}