objectiveai-cli 2.0.5

ObjectiveAI command-line interface and embeddable library
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

mod cli_test_util;

use std::path::{Path, PathBuf};
use std::process::Command;

fn snapshots_dir() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../objectiveai-api/assets/agent/completions/client_tests")
}

/// Extract the last assistant message content from a snapshot.
fn extract_assistant_content(snapshot: &serde_json::Value) -> String {
    let messages = snapshot["messages"].as_array().expect("snapshot should have messages array");
    messages.iter().rev()
        .find_map(|msg| {
            if msg["role"].as_str() == Some("assistant") {
                msg.get("content").and_then(|c| c.as_str()).map(String::from)
            } else {
                None
            }
        })
        .unwrap_or_default()
}

/// Run the CLI and return the assistant content emitted in its JSONL
/// stream. The CLI wraps output in `{begin}...{notification}...{end}`
/// lines; we walk the notifications and pluck out the one carrying the
/// `content` field.
fn run_cli_text(args: &[&str]) -> String {
    let mut cmd = Command::new(cli_test_util::cli_binary());
    cmd.env("CONFIG_BASE_DIR", cli_test_util::tests_dir());
    cmd.args(args);

    let output = cmd.output().expect("failed to execute CLI binary");
    let stdout = String::from_utf8_lossy(&output.stdout);
    let stderr = String::from_utf8_lossy(&output.stderr);

    if !output.status.success() {
        panic!(
            "CLI exited with {}\nargs: {:?}\nstdout: {stdout}\nstderr: {stderr}",
            output.status, args
        );
    }

    for line in stdout.lines() {
        let trimmed = line.trim();
        if trimmed.is_empty() || trimmed.starts_with("Logs ID:") {
            continue;
        }
        let Ok(value) = serde_json::from_str::<serde_json::Value>(trimmed) else {
            continue;
        };
        if value.get("type").and_then(|t| t.as_str()) != Some("notification") {
            continue;
        }
        if let Some(content) = value
            .pointer("/value/content")
            .and_then(|c| c.as_str())
        {
            return content.to_string();
        }
    }
    panic!("no `content` notification in CLI output:\n{stdout}");
}

macro_rules! snapshot_test {
    ($name:ident, $snapshot:expr, $messages_json:expr, $agent_name:expr, $seed:expr $(, $extra_args:expr)*) => {
        #[test]
        fn $name() {
            let seed_str = $seed.to_string();
            let agent_str = format!("remote=mock,name={}", $agent_name);
            let instructions_id = cli_test_util::instructions_id(
                cli_test_util::InstructionsScope::AgentCompletions,
            );
            let mut args = vec![
                "agents", "completions", "create", "standard",
                "--agent", &agent_str,
                "--messages-inline", $messages_json,
                "--seed", &seed_str,
                "--instructions-id", instructions_id.as_str(),
            ];
            $(
                for arg in $extra_args {
                    args.push(arg);
                }
            )*

            let actual = run_cli_text(&args);

            let snapshot = cli_test_util::load_snapshot(&snapshots_dir(), $snapshot);
            let expected = extract_assistant_content(&snapshot);

            assert_eq!(actual, expected, "assistant content mismatch for {}", $snapshot);
        }
    };
}

snapshot_test!(
    test_basic_mock_agent_seed_42,
    "test_basic_mock_agent_seed_42",
    "[]",
    "instruction",
    42
);

snapshot_test!(
    test_with_developer_and_user_messages,
    "test_with_developer_and_user_messages",
    r#"[{"role":"developer","content":"You are a helpful assistant."},{"role":"user","content":"What is 2+2?"}]"#,
    "instruction",
    99
);

snapshot_test!(
    test_json_object_response_format,
    "test_json_object_response_format",
    "[]",
    "instruction",
    42,
    &["--response-format-json-object"]
);