quvyta-cli 0.1.0-alpha.5

//! How a turn ends: whichever way a reply stops, and whatever a saved thread holds, the next
//! request is one the Messages API accepts. A `tool_use` whose `tool_result` never comes, or whose
//! `input` is not an object, is refused on that request and on every request after it, which
//! leaves the person a thread that cannot go on.
//!
//! What the turn cost is told here too: the provider counts the tokens each of its requests read
//! and wrote, a turn is many of them, and the totals under the conversation are the turn's own
//! rather than the conversation's.

use serde_json::{Value, json};

use super::tests::{Place, harness, read_card, say};
use super::*;
use crate::provider::fake::{FakeServer, Usage, calls, calls_using, says, says_using, sse};

/// Panics unless `body` is a request the API accepts as far as calls go: every `tool_use` has an
/// object for its input, and the message right after it is the person's and starts with a
/// `tool_result` for each of its calls.
fn assert_accepted(body: &Value) {
    let messages = body["messages"].as_array().expect("the messages");
    for (n, message) in messages.iter().enumerate() {
        let calls: Vec<&Value> = conversation::blocks(message).filter(|block| block["type"] == "tool_use").collect();
        if calls.is_empty() {
            continue;
        }
        let next = messages.get(n + 1).unwrap_or_else(|| panic!("message {n} has calls and nothing answers them"));
        assert_eq!(next["role"], "user", "the message after the calls of message {n} is the person's");
        let answered: Vec<&str> = conversation::blocks(next)
            .take_while(|block| block["type"] == "tool_result")
            .filter_map(|block| block["tool_use_id"].as_str())
            .collect();
        for call in calls {
            assert!(call["input"].is_object(), "the input of a call is an object: {call}");
            let id = call["id"].as_str().expect("an id");
            assert!(answered.contains(&id), "the call {id} of message {n} is answered first thing: {next}");
        }
    }
}

/// The sentence and the error flag of the result for call `id` in `body`.
fn result_in(body: &Value, id: &str) -> (String, bool) {
    body["messages"]
        .as_array()
        .into_iter()
        .flatten()
        .flat_map(conversation::blocks)
        .find(|block| block["type"] == "tool_result" && block["tool_use_id"] == id)
        .map(|block| (block["content"].as_str().unwrap_or_default().to_owned(), block["is_error"] == true))
        .unwrap_or_else(|| panic!("no result for {id} in {body}"))
}

/// A reply that says a little and starts a call `id` with the input fragment `partial`, then
/// stops for `reason`.
fn cut_call(id: &str, partial: &str, reason: &str) -> String {
    sse(&[
        json!({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),
        json!({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Reading it."}}),
        json!({"type": "content_block_stop", "index": 0}),
        json!({"type": "content_block_start", "index": 1, "content_block": {"type": "tool_use", "id": id, "name": "read_file", "input": {}}}),
        json!({"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": partial}}),
        json!({"type": "message_delta", "delta": {"stop_reason": reason}}),
        json!({"type": "message_stop"}),
    ])
}

#[test]
fn a_reply_cut_by_its_length_in_the_middle_of_a_call_leaves_a_thread_that_goes_on() {
    let place = Place::new("turn-cut-call");
    let server =
        FakeServer::start(vec![(200, cut_call("half", "{\"path\": \"src/ma", "max_tokens")), (200, says("Going on."))]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "read main");
    assert_eq!(harness.app().notice(), Some(&Notice::Cut));
    assert!(harness.screen().contains("reached its length limit"), "the person is told: {}", harness.screen());

    say(&mut harness, "go on");
    assert_eq!(server.received().len(), 2, "the next request went out");
    let body = &server.received()[1].body;
    assert_accepted(body);
    let (_, failed) = result_in(body, "half");
    assert!(failed, "a call cut in half is not done");
    assert!(harness.screen().contains("Going on."), "and it was answered: {}", harness.screen());
}

#[test]
fn a_reply_that_ends_its_turn_with_a_call_in_it_tells_the_model_the_call_was_not_taken() {
    let place = Place::new("turn-end-with-call");
    let server = FakeServer::start(vec![
        (200, cut_call("whole", "{\"path\": \"notes.txt\"}", "end_turn")),
        (200, says("Fine.")),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "look");
    assert_eq!(harness.app().phase(), &Phase::Idle, "the turn is over");
    say(&mut harness, "and?");
    let body = &server.received()[1].body;
    assert_accepted(body);
    assert_eq!(result_in(body, "whole"), (TURN_ENDED.to_owned(), true), "the model hears the turn ended first");
}

#[test]
fn a_person_who_stops_the_turn_is_named_as_the_one_who_stopped_it() {
    let place = Place::new("turn-person-stopped");
    let server = FakeServer::start(vec![
        (200, calls("r1", "run_command", &json!({"command": "printf a"}))),
        (200, says("Fine.")),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "run it");
    harness.press("esc");
    say(&mut harness, "never mind");
    let body = &server.received()[1].body;
    assert_accepted(body);
    assert_eq!(result_in(body, "r1"), (PERSON_STOPPED.to_owned(), true), "the model hears the person stopped it");
}

#[test]
fn a_call_whose_input_is_not_json_goes_back_as_an_object_that_still_holds_what_was_sent() {
    let place = Place::new("turn-bad-input");
    let server = FakeServer::start(vec![
        (200, cut_call("odd", "{\"path\": notes.txt}", "tool_use")),
        (200, says("Let me fix that.")),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "read the notes");
    assert_eq!(server.received().len(), 2, "the call is answered and the turn goes on");
    let body = &server.received()[1].body;
    assert_accepted(body);
    let call = body["messages"]
        .as_array()
        .into_iter()
        .flatten()
        .flat_map(conversation::blocks)
        .find(|block| block["type"] == "tool_use")
        .expect("the call");
    assert!(call["input"].to_string().contains("notes.txt"), "the model still sees what it sent: {}", call["input"]);
    assert!(result_in(body, "odd").1, "and hears it could not be done");
}

#[test]
fn a_thread_saved_broken_is_mended_when_the_folder_is_opened() {
    let place = Place::new("turn-saved-broken");
    // What an older qcli could leave: a call that was never answered, the person's next words
    // right after it, and an input kept as a string.
    let broken = json!({"version": 1, "messages": [
        {"role": "user", "content": [{"type": "text", "text": "read it"}]},
        {"role": "assistant", "content": [
            {"type": "text", "text": "Reading."},
            {"type": "tool_use", "id": "old", "name": "read_file", "input": "{\"path\": \"a"},
        ]},
        {"role": "user", "content": [{"type": "text", "text": "hello?"}]},
    ]});
    let file = place.store().file().to_path_buf();
    std::fs::create_dir_all(file.parent().expect("a folder")).expect("the conversations folder");
    std::fs::write(&file, broken.to_string()).expect("the saved thread");

    let server = FakeServer::start(vec![(200, says("Here again."))]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "are you there?");
    let body = &server.received()[0].body;
    assert_accepted(body);
    assert!(result_in(body, "old").1, "the old call is answered as not done");
    assert!(harness.screen().contains("Here again."), "{}", harness.screen());
}

#[test]
fn a_turn_stops_after_its_last_allowed_request_and_says_so() {
    let place = Place::new("turn-rounds");
    std::fs::write(place.work().join("a.txt"), "a\n").expect("a file");
    let answers: Vec<(u16, String)> =
        (0..=MAX_ROUNDS).map(|n| (200, calls(&format!("c{n}"), "read_file", &json!({"path": "a.txt"})))).collect();
    let server = FakeServer::start(answers);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "keep reading");
    assert_eq!(server.received().len(), MAX_ROUNDS, "no request past the last one allowed");
    assert_eq!(harness.app().notice(), Some(&Notice::Rounds));
    assert!(
        harness.screen().contains(&format!("stopped after {MAX_ROUNDS} requests")),
        "the person is told: {}",
        harness.screen()
    );
    assert_eq!(harness.app().phase(), &Phase::Idle, "and can write again");
}

#[test]
fn what_a_stopped_round_still_sends_back_is_not_taken() {
    let place = Place::new("turn-stale");
    let server = FakeServer::start(vec![
        (200, calls("r1", "run_command", &json!({"command": "printf a"}))),
        (200, says("Fine.")),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "run it");
    read_card(&mut harness);
    let stopped_round = harness.app().round;
    harness.press("esc");
    let kept = harness.app().conversation().clone();

    // What the task of the stopped round sends when it ends after the stop: its reply and the
    // outcome of its action.
    let outcome = crate::actions::Outcome { text: "late".to_owned(), is_error: false };
    harness.send(Msg::Done(stopped_round, "r1".to_owned(), outcome));
    harness.send(Msg::Replied(
        stopped_round,
        Ok(crate::provider::Reply {
            content: vec![json!({"type": "text", "text": "late reply"})],
            stop: crate::provider::Stop::EndTurn,
            input_tokens: None,
            output_tokens: None,
        }),
    ));
    assert_eq!(harness.app().conversation(), &kept, "nothing of the stopped round reaches the thread");
    assert!(!harness.screen().contains("late reply"));
    assert_eq!(harness.app().phase(), &Phase::Idle);
}

#[test]
fn an_action_stopped_while_it_is_being_done_is_not_told_as_not_done() {
    let place = Place::new("turn-stopped-while-doing");
    let server = FakeServer::start(vec![(200, says("I will check."))]);
    let mut app = place.app(Some(&server.address));
    // An edit being written when the person presses Esc: it may already be on the disk.
    let call = conversation::Call {
        id: "e1".to_owned(),
        name: "edit_file".to_owned(),
        input: json!({"path": "a.txt", "old_text": "", "new_text": "x"}),
    };
    app.conversation.say("write it");
    app.conversation.reply(vec![json!({"type": "tool_use", "id": "e1", "name": "edit_file", "input": call.input})]);
    let action = Action::from_call(&call.name, &call.input).expect("an edit");
    app.phase = Phase::Acting { call, action };
    let mut harness = harness(app);
    harness.press("esc");
    let (sentence, _) = result_in(&harness.app().conversation().to_json(), "e1");
    assert_eq!(sentence, STOPPED_WHILE_DOING, "the model is told it may have been done");
}

/// A turn of two requests, told on the screen as one sum. Each of the requests carries the whole
/// conversation, so what the person pays for is what both of them used together: 1000 and 1500
/// read, 20 and 30 written, and the numbers are ones no default of the stand-in would produce.
#[test]
fn a_turn_says_how_many_tokens_its_replies_read_and_wrote() {
    let place = Place::new("turn-tokens");
    std::fs::write(place.work().join("notes.txt"), "the secret word is apricot\n").expect("a file to read");
    let server = FakeServer::start(vec![
        (200, calls_using("c1", "read_file", &json!({"path": "notes.txt"}), Usage::new(1000, 20))),
        (200, says_using("The word is apricot.", Usage::new(1500, 30))),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "what is the word?");

    assert_eq!(server.received().len(), 2, "reading asks nothing, so the turn was two requests");
    let screen = harness.screen();
    assert!(screen.contains("This turn: 2500 tokens read, 50 written"), "both replies counted: {screen}");

    // What a turn cost is said on the screen and nowhere else: the thread the model reads, and the
    // file it is kept in, are not a place for the person's bill.
    let file = place.store().file().to_path_buf();
    let kept = std::fs::read_to_string(&file).expect("the conversation is saved");
    assert!(!kept.contains("2500") && !kept.contains("This turn"), "the totals are not kept: {kept}");
}

/// The totals are the turn's own: a new conversation forgets them, and the turn after it is
/// counted from nothing rather than added to what the last one spent.
#[test]
fn a_new_conversation_forgets_a_turns_totals_and_the_next_turn_counts_only_its_own() {
    let place = Place::new("turn-tokens-fresh");
    let server = FakeServer::start(vec![
        (200, says_using("First answer.", Usage::new(1000, 20))),
        (200, says_using("Second answer.", Usage::new(300, 5))),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "first question");
    assert!(harness.screen().contains("This turn: 1000 tokens read, 20 written"), "{}", harness.screen());

    harness.press("ctrl+n");
    assert!(
        !harness.screen().contains("This turn:"),
        "a new turn's line is gone with the old one: {}",
        harness.screen()
    );

    say(&mut harness, "second question");
    let screen = harness.screen();
    assert!(screen.contains("This turn: 300 tokens read, 5 written"), "only what this turn used: {screen}");
    assert!(!screen.contains("1300"), "the first turn's numbers are not added in: {screen}");
}

/// A provider that counted nothing is not one that counted zero, so there is nothing to say: the
/// line is for a turn the person was charged for.
#[test]
fn a_turn_whose_replies_count_nothing_says_no_totals() {
    let place = Place::new("turn-tokens-silent");
    let server = FakeServer::start(vec![(200, says_using("Nothing to count here.", Usage::SILENT))]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "hello");

    let screen = harness.screen();
    assert!(screen.contains("Nothing to count here."), "the answer is on the screen: {screen}");
    assert!(!screen.contains("This turn:"), "and a turn that used no token says nothing of it: {screen}");
}

/// The next message starts a turn of its own: its line says only what it used, not that and the
/// turn before it together.
#[test]
fn the_next_message_counts_its_turn_from_nothing() {
    let place = Place::new("turn-tokens-next");
    let server = FakeServer::start(vec![
        (200, says_using("First answer.", Usage::new(1000, 20))),
        (200, says_using("Second answer.", Usage::new(300, 5))),
    ]);
    let mut harness = harness(place.app(Some(&server.address)));
    say(&mut harness, "first question");
    say(&mut harness, "second question");

    let screen = harness.screen();
    assert!(screen.contains("This turn: 300 tokens read, 5 written"), "only this turn: {screen}");
    assert!(!screen.contains("1300"), "the first turn is not added in: {screen}");
}