quvyta-cli 0.1.0-alpha.5

//! What goes out: how big a request is, what of it is kept whole, what is left out in a note,
//! and what the person still has on disk afterwards. Every test drives the chat the way a person
//! does, from the keys, and reads what the stand-in server received.

use std::path::PathBuf;

use serde_json::{Value, json};

use super::tests::{Place, harness, say};
use super::*;
use crate::conversation::{self, Conversation};
use crate::instructions;
use crate::provider::fake::{FakeServer, calls, says};

/// A source file of about 100 KB, as big as the largest thing `read_file` brings back whole, with
/// a line of its own so one result can be told from another.
fn file(n: usize) -> String {
    let mut text = format!("// file number {n}\n");
    let line = "    let parts = input.split_whitespace().collect::<Vec<_>>();\n";
    while text.len() < 100 * 1024 {
        text.push_str(line);
    }
    text
}

/// The output of a build of about `bytes` bytes, as big as a busy `run_command` gets.
fn command(bytes: usize) -> String {
    let mut text = String::new();
    let mut n = 0;
    while text.len() < bytes {
        n += 1;
        text.push_str(&format!("   Compiling quvyta-cli v0.1.0-alpha.3 (/tmp/fake-build/qcli/src/file{n}.rs)\n"));
    }
    text
}

/// What the person asks in turn `n`. The first of them is the task, and the model reads a whole
/// file in every turn after it.
fn question(n: usize) -> String {
    format!("Turn {n}: read file{n}.rs and tell me what is in it.")
}

/// The stand-in's answers for `turns` turns of reading a whole file each.
fn answers(turns: usize) -> Vec<(u16, String)> {
    let mut answers = Vec::new();
    for n in 1..=turns {
        answers.push((200, calls(&format!("r{n}"), "read_file", &json!({"path": format!("file{n}.rs")}))));
        answers.push((200, says(&format!("Turn {n} is done."))));
    }
    answers
}

/// The files of `turns` turns, written where the model will find them.
fn folder(place: &Place, turns: usize) {
    for n in 1..=turns {
        std::fs::write(place.work().join(format!("file{n}.rs")), file(n)).expect("the file it reads");
    }
}

/// A long run driven from the keys: the person asks `turns` things, each one a read of a whole
/// 100 KB file, and the model answers each of them. What the server received, and the
/// conversation as it stands afterwards, is what the tests look at.
fn long_run(place: &Place, turns: usize) -> (FakeServer, Conversation) {
    folder(place, turns);
    let server = FakeServer::start(answers(turns));
    let mut harness = harness(place.app(Some(&server.address)));
    for n in 1..=turns {
        say(&mut harness, &question(n));
    }
    let conversation = harness.app().conversation().clone();
    (server, conversation)
}

/// The last message of a request body.
fn last_message(body: &Value) -> Value {
    body["messages"].as_array().and_then(|messages| messages.last().cloned()).expect("a message")
}

/// Every block of every message of a request body.
fn every_block(body: &Value) -> impl Iterator<Item = &Value> {
    body["messages"].as_array().into_iter().flatten().flat_map(conversation::blocks)
}

/// Whether a block is a note standing in for what was left out.
fn is_note(block: &Value) -> bool {
    block["content"].as_str().or_else(|| block["text"].as_str()).is_some_and(|text| text.starts_with("(Left out:"))
}

/// How many bytes a request body took, as the server read it.
fn size(request: &crate::provider::fake::Received) -> usize {
    serde_json::to_vec(&request.body).expect("the body is JSON").len()
}

/// The working folder as the program sees it, which is the canonical path.
fn root(place: &Place) -> PathBuf {
    std::fs::canonicalize(place.work()).expect("the working folder")
}

/// A reply that asks for two things at once, so that stopping it leaves calls waiting.
fn two_calls() -> String {
    crate::provider::fake::sse(&[
        json!({"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "x", "name": "run_command", "input": {}}}),
        json!({"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "{\"command\": \"printf x\"}"}}),
        json!({"type": "content_block_stop", "index": 0}),
        json!({"type": "content_block_start", "index": 1, "content_block": {"type": "tool_use", "id": "y", "name": "list_dir", "input": {}}}),
        json!({"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": "{\"path\": \".\"}"}}),
        json!({"type": "content_block_stop", "index": 1}),
        json!({"type": "message_delta", "delta": {"stop_reason": "tool_use"}}),
        json!({"type": "message_stop"}),
    ])
}

/// A long run of whole files: every request stays under the cap, and the size of each one is
/// printed so the numbers are in the test output.
#[test]
fn a_long_run_keeps_every_request_under_the_cap() {
    let place = Place::new("request-long");
    let (server, _) = long_run(&place, 8);
    let received = server.received();
    assert_eq!(received.len(), 16, "two requests a turn: the question, and the answer to the result");

    let mut biggest = 0;
    for (n, request) in received.iter().enumerate() {
        let bytes = size(request);
        println!("request {n}: {bytes} bytes, about {} tokens (estimate, bytes/4)", bytes / 4);
        assert!(bytes < instructions::REQUEST_CAP, "request {n} is {bytes} bytes: over the cap");
        biggest = biggest.max(bytes);
    }
    println!("biggest of the run: {biggest} bytes");

    let kept = std::fs::read_to_string(place.store().file()).expect("the conversation is saved");
    assert!(kept.len() > instructions::REQUEST_CAP, "the conversation itself is {} bytes", kept.len());
}

/// The newest turn goes out as it is, and so does every result the model is waiting for an
/// answer to.
#[test]
fn the_newest_turn_and_the_results_it_waits_for_are_whole_in_every_request() {
    let place = Place::new("request-newest");
    let (server, conversation) = long_run(&place, 6);
    let stored = conversation.messages();
    for (n, request) in server.received().iter().enumerate() {
        let sent = request.body["messages"].as_array().expect("messages");
        assert_eq!(
            sent[0], stored[0],
            "the first message of request {n} is whole, and nothing is dropped in front of it"
        );
        for block in conversation::blocks(&last_message(&request.body)) {
            if block["type"] == "tool_result" {
                assert!(!is_note(block), "request {n} left out a result the model waits for");
            }
        }
    }

    let newest = last_message(&server.received().last().expect("requests").body);
    let result = conversation::blocks(&newest).find(|block| block["type"] == "tool_result").expect("the newest result");
    assert!(
        result["content"].as_str().is_some_and(|text| text.contains("// file number 6")),
        "the newest read goes out whole: {result}"
    );
}

/// The system prompt and the folder's `AGENTS.md` are what the model is, and go out whole.
#[test]
fn the_system_prompt_and_the_agents_file_are_whole_in_every_request() {
    let place = Place::new("request-system");
    std::fs::write(place.work().join("AGENTS.md"), "Answer like a pirate, and say arr.\n").expect("write");
    let (server, _) = long_run(&place, 6);
    for request in server.received() {
        let system = request.body["system"].as_str().expect("a system prompt");
        assert!(system.contains("You are qcli"), "{system}");
        assert!(system.ends_with("Answer like a pirate, and say arr."), "{system}");
    }
}

/// Where something was left out a note says so, and what is kept is kept whole.
#[test]
fn what_is_left_out_stands_in_a_note_and_what_is_kept_stays_whole() {
    let place = Place::new("request-note");
    let (server, _) = long_run(&place, 6);
    let received = server.received();
    let body = &received.last().expect("requests").body;

    assert_eq!(body["messages"][0]["content"][0]["text"], question(1), "the first message is never touched");
    let newest = serde_json::to_string(&last_message(body)).expect("JSON");
    assert!(newest.contains("// file number 6"), "the newest read is whole, line and all: {newest}");

    let notes: Vec<&str> = every_block(body)
        .filter(|block| block["type"] == "tool_result" && is_note(block))
        .filter_map(|block| block["content"].as_str())
        .collect();
    assert!(!notes.is_empty(), "six turns of whole files left something out, or this proves nothing");
    for note in notes {
        assert!(note.contains("bytes of output from read_file"), "{note}");
        assert!(note.contains("The whole of it is in the conversation saved for this folder."), "{note}");
    }
}

/// A conversation that fits the cap goes out exactly as it did before the cap existed: what the
/// untrimmed builder makes of it and what went out are the same bytes.
#[test]
fn a_short_conversation_goes_out_byte_for_byte_as_it_did() {
    let place = Place::new("request-short");
    let server = FakeServer::start(vec![(200, says("Short answer."))]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "hello there");

    // The transport adds the model and that it is a stream; the body itself is what the cap is
    // about, so those two are taken off before the bytes are compared.
    let mut sent = server.received()[0].body.clone();
    let sent = sent.as_object_mut().expect("an object");
    sent.remove("model");
    sent.remove("stream");

    // The conversation as it stood when that request went out: what the person had said, before
    // the answer came back and joined it.
    let mut asked = Conversation::default();
    asked.say("hello there");
    let system = instructions::system(&root(&place));
    let trimmed = serde_json::to_vec(&instructions::body(&asked, &system)).expect("JSON");
    let untrimmed = serde_json::to_vec(&instructions::body_within(&asked, &system, usize::MAX)).expect("JSON");

    assert_eq!(trimmed, untrimmed, "the cap does not touch a conversation that fits it");
    assert_eq!(serde_json::to_vec(&sent).expect("JSON"), trimmed, "and what went out is that very body");
}

/// What a note says is in the conversation on disk, so the note is not a guess: the person can
/// open the folder tomorrow and read all of it.
#[test]
fn the_conversation_on_disk_still_holds_what_the_request_left_out() {
    let place = Place::new("request-saved");
    let (server, _) = long_run(&place, 6);
    let left_out =
        every_block(&server.received().last().expect("requests").body).filter(|block| is_note(block)).count();
    assert!(left_out > 0, "this run left something out, or the test proves nothing");

    let saved = std::fs::read_to_string(place.store().file()).expect("the conversation is saved");
    for n in 1..=6 {
        assert!(saved.contains(&format!("// file number {n}")), "turn {n} is still there to read");
    }
}

/// No request goes out with a call the model has not answered: a turn that ends in a call waiting
/// for an answer is never sent, and shortening a result does not lose the call it belongs to.
#[test]
fn no_request_goes_out_with_a_call_that_has_no_answer() {
    let place = Place::new("request-answered");
    let (server, _) = long_run(&place, 6);
    for (n, request) in server.received().iter().enumerate() {
        let asked: Vec<&str> = every_block(&request.body)
            .filter(|block| block["type"] == "tool_use")
            .filter_map(|block| block["id"].as_str())
            .collect();
        let answered: Vec<&str> = every_block(&request.body)
            .filter(|block| block["type"] == "tool_result")
            .filter_map(|block| block["tool_use_id"].as_str())
            .collect();
        for id in asked {
            assert!(answered.contains(&id), "request {n} asks for {id} and never answers it");
        }
    }
}

/// Stopped half way in a long conversation, the turn still goes out whole and answered, and still
/// under the cap.
#[test]
fn a_turn_stopped_half_way_still_goes_out_whole_and_under_the_cap() {
    let place = Place::new("request-stopped");
    folder(&place, 5);
    let mut answers = answers(5);
    answers.push((200, two_calls()));
    answers.push((200, says("Fine.")));
    let server = FakeServer::start(answers);
    let mut harness = harness(place.app(Some(&server.address)));
    for n in 1..=5 {
        say(&mut harness, &question(n));
    }

    say(&mut harness, "Do two things at once.");
    assert!(matches!(harness.app().phase(), Phase::Asking { .. }));
    harness.press("esc");
    assert_eq!(harness.app().notice(), Some(&Notice::Stopped));
    say(&mut harness, "go on");

    let received = server.received();
    assert_eq!(received.len(), 12, "nothing more went out while the turn was stopped");
    for (n, request) in received.iter().enumerate() {
        assert!(size(request) < instructions::REQUEST_CAP, "request {n} is over the cap");
    }
    let last = last_message(&received.last().expect("requests").body);
    let ids: Vec<&str> = conversation::blocks(&last)
        .filter(|block| block["content"] == PERSON_STOPPED)
        .map(|block| block["tool_use_id"].as_str().unwrap_or_default())
        .collect();
    assert_eq!(ids, ["x", "y"], "both waiting calls are answered, and whole: {last}");
}

/// A conversation of a morning's work: sixty turns of whole files, builds and long answers, built
/// in memory and never written in the repository.
fn morning(turns: usize) -> Conversation {
    let mut conversation = Conversation::default();
    conversation.say("Read the folder, fix what is broken and tell me what you changed. Start with the module that has the most tests.");
    for n in 1..=turns {
        let id = format!("call{n}");
        let (name, input, content) = if n % 3 == 0 {
            ("run_command", json!({"command": "cargo test --workspace"}), command(30 * 1024))
        } else {
            ("read_file", json!({"path": format!("src/app/file{n}.rs")}), file(n))
        };
        let answer = if n % 5 == 0 {
            format!("Turn {n}: here is what the code says. {}", "The count is kept as it was given. ".repeat(80))
        } else {
            format!("Turn {n}: it is ordinary code and nothing is wrong with it.")
        };
        conversation.reply(vec![
            json!({"type": "text", "text": answer}),
            json!({"type": "tool_use", "id": id, "name": name, "input": input}),
        ]);
        conversation.results(vec![conversation::result(&id, &content, false)]);
    }
    conversation.say("That is enough for now, thank you.");
    conversation
}

/// The measurement the design document records: the same sixty turns of work before and after the
/// cap, in bytes and in a rough token count.
#[test]
fn a_mornings_work_before_and_after_the_cap() {
    let place = Place::new("request-measure");
    let conversation = morning(60);
    let system = instructions::system(&root(&place));
    let before = serde_json::to_string(&instructions::body_within(&conversation, &system, usize::MAX)).expect("JSON");
    let after = serde_json::to_string(&instructions::body(&conversation, &system)).expect("JSON");

    println!(
        "before the cap: {} bytes, about {} tokens (estimate, bytes/4), {} messages",
        before.len(),
        before.len() / 4,
        conversation.messages().len()
    );
    println!("after the cap:  {} bytes, about {} tokens (estimate, bytes/4)", after.len(), after.len() / 4);
    println!("the cap:        {} bytes", instructions::REQUEST_CAP);
    let blocks = |wanted: &str| -> usize {
        conversation
            .messages()
            .iter()
            .flat_map(conversation::blocks)
            .filter(|block| block["type"] == wanted)
            .map(|block| serde_json::to_string(block).expect("JSON").len())
            .sum()
    };
    println!(
        "of the before: system {} bytes, tools {} bytes, tool_result {} bytes, text {} bytes, tool_use {} bytes",
        instructions::system(&root(&place)).len(),
        crate::actions::Action::definitions().to_string().len(),
        blocks("tool_result"),
        blocks("text"),
        blocks("tool_use")
    );

    assert!(before.len() > instructions::REQUEST_CAP, "a morning's work is over the cap: {} bytes", before.len());
    assert!(after.len() < instructions::REQUEST_CAP, "the cap holds: {} bytes", after.len());
    assert!(before.len() / after.len() > 4, "the request is at least four times smaller");
}