quvyta-cli 0.1.0-alpha.5

//! The three looking tools driven the way a person drives them: the person asks a question and
//! presses Enter, the stand-in provider answers with a tool call that carries the new fields, the
//! action is done and its result goes back to the model. What the model receives and what the
//! person sees are both checked, because that is the whole of the work these tools do.

use std::fmt::Write as _;

use serde_json::json;

use super::tests::{Place, harness, say};
use crate::provider::fake::{FakeServer, calls, says};

/// A file of `total` lines, each one saying which line it is, in the working folder of `place`.
fn long_file(place: &Place, name: &str, total: usize) -> String {
    let text: String = (1..=total).fold(String::new(), |mut text, number| {
        let _ = writeln!(text, "line {number}");
        text
    });
    std::fs::write(place.work().join(name), &text).expect("write the long file");
    text
}

/// What the model received as the result of the call `id` in the request numbered `index`: the
/// text and whether it failed.
fn result_of(server: &FakeServer, index: usize, id: &str) -> (String, bool) {
    let request = &server.received()[index].body;
    let message = request["messages"].as_array().expect("messages").last().expect("a message");
    let block = message["content"]
        .as_array()
        .expect("blocks")
        .iter()
        .find(|block| block["tool_use_id"] == id)
        .unwrap_or_else(|| panic!("no result for the call {id}: {message}"));
    (block["content"].as_str().expect("text").to_owned(), block["is_error"] == json!(true))
}

#[test]
fn a_read_of_a_range_in_the_middle_of_a_long_file_gives_those_lines() {
    let place = Place::new("read-range");
    long_file(&place, "long.txt", 3000);
    let server = FakeServer::start(vec![
        (200, calls("r1", "read_file", &json!({ "path": "long.txt", "start_line": 500, "end_line": 700 }))),
        (200, says("Lines 500 to 700 are about lines 500 to 700.")),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "what is on lines 500 to 700 of long.txt?");

    let (text, is_error) = result_of(&server, 1, "r1");
    assert!(!is_error, "{text}");
    let lines: Vec<&str> = text.lines().collect();
    assert_eq!(lines.len(), 202, "the 201 lines asked for and the note under them");
    assert_eq!(lines[0], "line 500");
    assert_eq!(lines[1], "line 501");
    assert_eq!(lines[200], "line 700");
    assert_eq!(lines[201], "(Lines 500-700 of 3000. Not shown: 499 lines above and 2300 lines below.)");
    assert!(!text.contains("line 499"), "and nothing above the range");
    assert!(!text.contains("line 701"), "nor below it");

    // The person sees the same one line in the transcript as any other read.
    let screen = harness.screen();
    assert!(screen.contains("read long.txt"), "{screen}");
    assert!(screen.contains("line 500"), "and the start of the answer: {screen}");
}

#[test]
fn a_read_without_a_range_is_the_head_of_the_file_as_it_always_was() {
    let place = Place::new("read-head");
    long_file(&place, "long.txt", 3000);
    let server = FakeServer::start(vec![
        (200, calls("r1", "read_file", &json!({ "path": "long.txt" }))),
        (200, says("I read the top of it.")),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "read long.txt");

    let (text, is_error) = result_of(&server, 1, "r1");
    assert!(!is_error, "{text}");
    assert!(text.contains("line 1\n") && text.contains("line 2000\n"), "the first 2000 lines");
    assert!(!text.contains("line 2001\n"), "and no more");
    assert!(
        text.ends_with("(Cut: the file has 3000 lines; only the first 2000 are shown.)"),
        "the note a read without a range has always had: {}",
        &text[text.len() - 80..]
    );
    assert!(harness.screen().contains("read long.txt"), "{}", harness.screen());
}

#[test]
fn a_range_that_cannot_be_used_is_an_error_the_person_sees() {
    let place = Place::new("read-bad");
    let file = long_file(&place, "long.txt", 3000);
    let server = FakeServer::start(vec![
        (
            200,
            crate::provider::fake::sse(&[
                json!({"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "a", "name": "read_file", "input": {}}}),
                json!({"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "{\"path\": \"long.txt\", \"start_line\": 3001, \"end_line\": 4000}"}}),
                json!({"type": "content_block_stop", "index": 0}),
                json!({"type": "content_block_start", "index": 1, "content_block": {"type": "tool_use", "id": "b", "name": "read_file", "input": {}}}),
                json!({"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": "{\"path\": \"long.txt\", \"start_line\": 700, \"end_line\": 500}"}}),
                json!({"type": "content_block_stop", "index": 1}),
                json!({"type": "content_block_start", "index": 2, "content_block": {"type": "tool_use", "id": "c", "name": "read_file", "input": {}}}),
                json!({"type": "content_block_delta", "index": 2, "delta": {"type": "input_json_delta", "partial_json": "{\"path\": \"long.txt\", \"start_line\": 0, \"end_line\": 10}"}}),
                json!({"type": "content_block_stop", "index": 2}),
                json!({"type": "message_delta", "delta": {"stop_reason": "tool_use"}}),
                json!({"type": "message_stop"}),
            ]),
        ),
        (200, says("Those ranges are not ones I can use.")),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "read me the last lines of long.txt");

    let (past, error) = result_of(&server, 1, "a");
    assert!(error, "{past}");
    assert_eq!(past, "The file has 3000 lines, so `start_line` 3001 is past its end.");
    let (inverted, error) = result_of(&server, 1, "b");
    assert!(error, "{inverted}");
    assert!(inverted.contains("`end_line` is before `start_line`"), "{inverted}");
    let (zero, error) = result_of(&server, 1, "c");
    assert!(error, "{zero}");
    assert_eq!(zero, "Lines are counted from 1, so `start_line` must be 1 or more.");

    // The person is told, and the file on disk is as it was.
    let screen = harness.screen();
    assert!(screen.contains("past its end"), "{screen}");
    assert!(screen.contains("before `start_line`"), "{screen}");
    assert_eq!(std::fs::read_to_string(place.work().join("long.txt")).expect("read"), file);
}

#[test]
fn a_listing_with_a_depth_is_a_tree_and_a_deeper_one_names_the_maximum() {
    let place = Place::new("list-depth");
    std::fs::create_dir_all(place.work().join("src/deep")).expect("folder");
    std::fs::write(place.work().join("src/one.txt"), "0123456789").expect("write");
    std::fs::write(place.work().join("src/deep/two.md"), "x".repeat(2500)).expect("write");
    std::fs::write(place.work().join("top.txt"), "top").expect("write");
    let server = FakeServer::start(vec![
        (
            200,
            crate::provider::fake::sse(&[
                json!({"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "a", "name": "list_dir", "input": {}}}),
                json!({"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "{\"path\": \"\", \"depth\": 2}"}}),
                json!({"type": "content_block_stop", "index": 0}),
                json!({"type": "content_block_start", "index": 1, "content_block": {"type": "tool_use", "id": "b", "name": "list_dir", "input": {}}}),
                json!({"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": "{\"path\": \"\", \"depth\": 9}"}}),
                json!({"type": "content_block_stop", "index": 1}),
                json!({"type": "message_delta", "delta": {"stop_reason": "tool_use"}}),
                json!({"type": "message_stop"}),
            ]),
        ),
        (200, says("The folder has src and top.txt in it.")),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "what is in this folder?");

    let (tree, is_error) = result_of(&server, 1, "a");
    assert!(!is_error, "{tree}");
    assert_eq!(
        tree,
        "src/\n\
         \x20 deep/\n\
         \x20 one.txt  10 B\n\
         top.txt    3 B",
        "an indented tree, folders before files, with the size of every file"
    );
    let (deep, is_error) = result_of(&server, 1, "b");
    assert!(is_error, "{deep}");
    assert_eq!(deep, "A depth of 9 is not one; `list_dir` takes a depth from 1 to 3.");

    let screen = harness.screen();
    assert!(screen.contains("one.txt"), "the tree is on the screen: {screen}");
    assert!(screen.contains("10 B"), "with the sizes: {screen}");
    assert!(screen.contains("from 1 to 3"), "and the refusal: {screen}");
}

#[test]
fn a_search_can_ignore_case_and_be_narrowed_by_a_glob() {
    let place = Place::new("search-narrow");
    std::fs::create_dir_all(place.work().join("src")).expect("folder");
    std::fs::create_dir_all(place.work().join("docs")).expect("folder");
    std::fs::write(place.work().join("src/main.rs"), "let widget = 1;\n").expect("write");
    std::fs::write(place.work().join("docs/guide.md"), "the Widget is here\n").expect("write");
    let server = FakeServer::start(vec![
        (
            200,
            crate::provider::fake::sse(&[
                json!({"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "a", "name": "search", "input": {}}}),
                json!({"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "{\"pattern\": \"widget\", \"ignore_case\": true}"}}),
                json!({"type": "content_block_stop", "index": 0}),
                json!({"type": "content_block_start", "index": 1, "content_block": {"type": "tool_use", "id": "b", "name": "search", "input": {}}}),
                json!({"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": "{\"pattern\": \"widget\", \"glob\": \"src/*\"}"}}),
                json!({"type": "content_block_stop", "index": 1}),
                json!({"type": "content_block_start", "index": 2, "content_block": {"type": "tool_use", "id": "c", "name": "search", "input": {}}}),
                json!({"type": "content_block_delta", "index": 2, "delta": {"type": "input_json_delta", "partial_json": "{\"pattern\": \"gadget\", \"ignore_case\": true, \"glob\": \"*.md\"}"}}),
                json!({"type": "content_block_stop", "index": 2}),
                json!({"type": "message_delta", "delta": {"stop_reason": "tool_use"}}),
                json!({"type": "message_stop"}),
            ]),
        ),
        (200, says("Two files mention a widget.")),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "where is a widget?");

    // The flag finds `Widget` when asked for `widget`, in both spellings.
    let (any_case, is_error) = result_of(&server, 1, "a");
    assert!(!is_error, "{any_case}");
    assert_eq!(any_case, "docs/guide.md:1: the Widget is here\nsrc/main.rs:1: let widget = 1;");
    // The glob narrows the walk to the one folder.
    let (one_folder, is_error) = result_of(&server, 1, "b");
    assert!(!is_error, "{one_folder}");
    assert_eq!(one_folder, "src/main.rs:1: let widget = 1;");
    // Nothing found names the two ways to try again.
    let (none, is_error) = result_of(&server, 1, "c");
    assert!(!is_error, "{none}");
    assert_eq!(none, "No matches for `gadget` in `.`. Try `ignore_case` for the same text in another case.");

    let screen = harness.screen();
    assert!(screen.contains("search widget"), "{screen}");
    assert!(screen.contains("main.rs:1"), "the person follows the matches: {screen}");
}

#[test]
fn the_request_carries_the_five_tools_with_the_new_fields() {
    let place = Place::new("tools");
    let server = FakeServer::start(vec![(200, says("Hello."))]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "hello");

    let request = &server.received()[0].body;
    let tools = request["tools"].as_array().expect("the five tools");
    assert_eq!(tools.len(), 5, "the five tools are still five");
    let names: Vec<&str> = tools.iter().map(|tool| tool["name"].as_str().expect("a name")).collect();
    assert_eq!(names, ["read_file", "list_dir", "search", "edit_file", "run_command"]);
    let field = |tool: &str, name: &str| {
        let schema = &tools[tools.iter().position(|entry| entry["name"] == tool).expect("a tool")]["input_schema"];
        assert_eq!(schema["properties"][name]["type"], tool_type_of(name), "{tool}.{name}");
        schema["properties"][name]["description"].as_str().expect("a description").to_owned()
    };
    for (tool, name) in [("read_file", "start_line"), ("read_file", "end_line"), ("list_dir", "depth")] {
        assert!(field(tool, name).contains("500") || field(tool, name).contains("2"), "{tool}.{name} has no example");
    }
    assert!(field("search", "glob").contains("src/*.rs"), "the glob names an example");
    assert!(field("search", "ignore_case").contains("Widget"), "the flag names an example");
    // Each tool still asks for only what it always asked for, so an old call still parses.
    assert_eq!(tools[0]["input_schema"]["required"], json!(["path"]));
    assert_eq!(tools[2]["input_schema"]["required"], json!(["pattern"]));

    // The system prompt tells the model how to use them.
    let system = request["system"].as_str().expect("the system prompt");
    for said in ["list_dir", "start_line", "end_line", "ignore_case", "glob"] {
        assert!(system.contains(said), "`{said}` is not in the prompt:\n{system}");
    }
    assert!(harness.screen().contains("Hello."), "{}", harness.screen());
}

/// The JSON type a new field is given in the manual.
fn tool_type_of(name: &str) -> &'static str {
    match name {
        "ignore_case" => "boolean",
        "glob" => "string",
        _ => "integer",
    }
}

#[test]
fn a_new_field_does_not_open_a_way_out_of_the_working_folder() {
    let place = Place::new("confine-fields");
    // The test's own folder, which holds the working folder and is removed with it, is where a
    // file outside the working folder can be written.
    let outside_folder = place.work().parent().expect("the test's own folder").to_owned();
    let outside = outside_folder.join("secret.txt");
    std::fs::write(&outside, "the secret is out\n").expect("write outside");
    let server = FakeServer::start(vec![
        (
            200,
            crate::provider::fake::sse(&[
                json!({"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "a", "name": "read_file", "input": {}}}),
                json!({"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "{\"path\": \"../secret.txt\", \"start_line\": 1, \"end_line\": 1}"}}),
                json!({"type": "content_block_stop", "index": 0}),
                json!({"type": "content_block_start", "index": 1, "content_block": {"type": "tool_use", "id": "b", "name": "search", "input": {}}}),
                json!({"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": "{\"pattern\": \"secret\", \"path\": \"..\", \"glob\": \"*\"}"}}),
                json!({"type": "content_block_stop", "index": 1}),
                json!({"type": "content_block_start", "index": 2, "content_block": {"type": "tool_use", "id": "c", "name": "search", "input": {}}}),
                json!({"type": "content_block_delta", "index": 2, "delta": {"type": "input_json_delta", "partial_json": "{\"pattern\": \"secret\", \"glob\": \"../*\"}"}}),
                json!({"type": "content_block_stop", "index": 2}),
                json!({"type": "message_delta", "delta": {"stop_reason": "tool_use"}}),
                json!({"type": "message_stop"}),
            ]),
        ),
        (200, says("I cannot read outside the folder.")),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "read the secret above this folder");

    // Today's sentence, for a range outside and for a glob beside it.
    let (read_out, is_error) = result_of(&server, 1, "a");
    assert!(is_error, "{read_out}");
    assert_eq!(read_out, "`../secret.txt` is outside the working folder; only paths inside it can be used.");
    let (search_out, is_error) = result_of(&server, 1, "b");
    assert!(is_error, "{search_out}");
    assert!(search_out.contains("outside the working folder"), "{search_out}");
    // A glob is a name pattern, not a path: it cannot reach out of the folder either.
    let (glob_out, is_error) = result_of(&server, 1, "c");
    assert!(!is_error, "{glob_out}");
    assert!(glob_out.starts_with("No matches for `secret` in `.`."), "{glob_out}");

    assert_eq!(std::fs::read_to_string(&outside).expect("read outside"), "the secret is out\n", "nothing was read");
    assert!(harness.screen().contains("outside the working folder"), "{}", harness.screen());
}

#[test]
fn an_empty_folder_and_a_file_that_is_not_text_give_a_sentence() {
    let place = Place::new("awkward");
    std::fs::create_dir_all(place.work().join("empty")).expect("folder");
    std::fs::write(place.work().join("image.bin"), b"\x89PNG\r\n\x1a\n\0\0binary").expect("write");
    let server = FakeServer::start(vec![
        (
            200,
            crate::provider::fake::sse(&[
                json!({"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "a", "name": "list_dir", "input": {}}}),
                json!({"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "{\"path\": \"empty\", \"depth\": 3}"}}),
                json!({"type": "content_block_stop", "index": 0}),
                json!({"type": "content_block_start", "index": 1, "content_block": {"type": "tool_use", "id": "b", "name": "read_file", "input": {}}}),
                json!({"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": "{\"path\": \"image.bin\", \"start_line\": 1, \"end_line\": 2}"}}),
                json!({"type": "content_block_stop", "index": 1}),
                json!({"type": "message_delta", "delta": {"stop_reason": "tool_use"}}),
                json!({"type": "message_stop"}),
            ]),
        ),
        (200, says("The folder is empty and the file is not text.")),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    // The folder has no AGENTS.md of its own, and qcli's own instructions go out all the same.
    assert!(harness.screen().contains("Ask anything about this folder"), "{}", harness.screen());
    say(&mut harness, "what is in empty, and what is in image.bin?");

    let (empty, is_error) = result_of(&server, 1, "a");
    assert!(!is_error, "{empty}");
    assert_eq!(empty, "`empty` is empty.");
    let (binary, is_error) = result_of(&server, 1, "b");
    assert!(is_error, "{binary}");
    assert!(binary.contains("is not a text file"), "{binary}");

    let system = server.received()[0].body["system"].as_str().expect("the system prompt").to_owned();
    assert!(system.contains("You are qcli"), "{system}");
    assert!(!system.contains("AGENTS.md says"), "the folder has no instructions of its own: {system}");
    let screen = harness.screen();
    assert!(screen.contains("is empty"), "{screen}");
    assert!(screen.contains("is not a text file"), "{screen}");
}