objectiveai-cli 2.0.5

ObjectiveAI command-line interface and embeddable library
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

mod cli_test_util;

use std::path::{Path, PathBuf};

fn snapshots_dir() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../objectiveai-api/assets/functions/executions/client_tests")
}

fn snapshot_output(snapshot: &serde_json::Value) -> serde_json::Value {
    snapshot["output"]["output"].clone()
}

fn snapshot_has_errors(snapshot: &serde_json::Value) -> bool {
    snapshot["tasks_errors"].as_bool().unwrap_or(false)
}

macro_rules! snapshot_test {
    ($name:ident, $snapshot:expr, $function:expr, $profile:expr, $seed:expr, $input:tt) => {
        #[test]
        fn $name() {
            let input = serde_json::to_string(&serde_json::json!($input)).unwrap();
            let seed_str = $seed.to_string();

            let function_str = format!("remote=mock,name={}", $function);
            let profile_str = format!("remote=mock,name={}", $profile);
            let instructions_id = cli_test_util::instructions_id(
                cli_test_util::InstructionsScope::FunctionExecutions,
            );
            let cli_result = cli_test_util::run_cli(&[
                "functions", "executions", "create", "standard",
                "--function", &function_str,
                "--profile", &profile_str,
                "--input-inline", &input,
                "--seed", &seed_str,
                "--instructions-id", instructions_id.as_str(),
            ]);

            let snapshot = cli_test_util::load_snapshot(&snapshots_dir(), $snapshot);
            let expected_output = cli_test_util::rounded(&snapshot_output(&snapshot));
            let has_errors = snapshot_has_errors(&snapshot);

            let actual_output = cli_test_util::rounded(&cli_result["output"]);
            assert_eq!(actual_output, expected_output, "output mismatch for {}", $snapshot);

            if has_errors {
                assert!(
                    cli_result.get("errors").is_some_and(|e| e.as_array().is_some_and(|a| !a.is_empty())),
                    "expected errors for {} but got none", $snapshot
                );
            } else {
                assert!(
                    cli_result.get("errors").is_none()
                        || cli_result["errors"].as_array().is_some_and(|a| a.is_empty()),
                    "expected no errors for {} but got: {:?}", $snapshot, cli_result.get("errors")
                );
            }
        }
    };
}

snapshot_test!(
    mock_1_scalar_leaf_binary_seed_42,
    "mock_1_scalar_leaf_binary_seed_42",
    "binary-classifier",
    "solo-instruction",
    42,
    {"text": "Hello world"}
);

snapshot_test!(
    mock_7_vector_5_criteria_seed_42,
    "mock_7_vector_5_criteria_seed_42",
    "five-criteria-ranker",
    "schema-heavy-trio",
    42,
    {"items": ["Option A", "Option B", "Option C"]}
);

snapshot_test!(
    mock_20_vector_super_branch_seed_42,
    "mock_20_vector_super_branch_seed_42",
    "nested-vector-super-branch",
    "nested-vector-inline-remote",
    42,
    {"items": ["Alpha", "Beta", "Gamma"]}
);

/// Split: tweet-scorer over 10 real tweets (input loaded from
/// `inputs/10_tweets.json`), seed 42. Mirrors the Rust api test
/// `test_split_tweet_scorer_10_tweets_seed_42`: same mock function,
/// same inline profile (two mock instruction agents, one with
/// top_logprobs=6, equal weights), same input file, same seed.
#[test]
fn split_tweet_scorer_10_tweets_seed_42() {
    let snapshots = snapshots_dir();
    let input_path = snapshots.join("inputs/10_tweets.json");
    let input = std::fs::read_to_string(&input_path)
        .unwrap_or_else(|e| panic!("failed to read {}: {e}", input_path.display()));

    let profile_inline = r#"{
        "agents": [
          {"count": 1, "upstream": "mock", "output_mode": "instruction", "top_logprobs": 6},
          {"count": 1, "upstream": "mock", "output_mode": "instruction"}
        ],
        "weights": [1.0, 1.0]
    }"#;

    let id = cli_test_util::instructions_id(
        cli_test_util::InstructionsScope::FunctionExecutions,
    );
    let cli_result = cli_test_util::run_cli(&[
        "functions", "executions", "create", "standard",
        "--function", "remote=mock,name=tweet-scorer",
        "--profile-inline", profile_inline,
        "--input-inline", &input,
        "--split",
        "--seed", "42",
        "--instructions-id", id.as_str(),
    ]);

    let snapshot = cli_test_util::load_snapshot(&snapshots, "split_tweet_scorer_10_tweets_seed_42");
    let expected_output = cli_test_util::rounded(&snapshot_output(&snapshot));
    let has_errors = snapshot_has_errors(&snapshot);

    let actual_output = cli_test_util::rounded(&cli_result["output"]);
    assert_eq!(actual_output, expected_output, "output mismatch for split_tweet_scorer_10_tweets_seed_42");

    if has_errors {
        assert!(
            cli_result.get("errors").is_some_and(|e| e.as_array().is_some_and(|a| !a.is_empty())),
            "expected errors but got none",
        );
    } else {
        assert!(
            cli_result.get("errors").is_none()
                || cli_result["errors"].as_array().is_some_and(|a| a.is_empty()),
            "expected no errors but got: {:?}", cli_result.get("errors"),
        );
    }
}