llm-browser-testkit 0.2.5

LLM-driven browser test framework — define browser test scenarios in TOML with natural language steps and LLM-powered assertions
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

//! Test that temperature, thinking, and `model_params` are parsed from TOML.

#![allow(
    clippy::expect_used,
    clippy::unwrap_used,
    clippy::panic,
    clippy::missing_panics_doc
)]

use llm_browser_testkit::scenario::Scenario;

#[test]
fn test_toml_parses_temperature_and_thinking() {
    let toml = r#"
[config]
temperature = 0.7
thinking = true

[[definitions]]
name = "no_errors"
preset = "no_error_on_page"

[[test]]
name = "dummy"
steps = []
"#;

    let scenario: Scenario = toml::from_str(toml).expect("parse TOML");
    assert!((scenario.config.temperature - 0.7).abs() < f64::EPSILON);
    assert_eq!(scenario.config.thinking, Some(true));
}

#[test]
fn test_default_temperature_and_thinking() {
    let toml = r#"
[[definitions]]
name = "no_errors"
preset = "no_error_on_page"

[[test]]
name = "dummy"
steps = []
"#;

    let scenario: Scenario = toml::from_str(toml).expect("parse TOML");
    assert!((scenario.config.temperature - 0.0).abs() < f64::EPSILON);
    assert_eq!(scenario.config.thinking, None);
}

#[test]
fn test_temperature_zero_and_thinking_false() {
    let toml = r#"
[config]
temperature = 0.0
thinking = false

[[definitions]]
name = "no_errors"
preset = "no_error_on_page"

[[test]]
name = "dummy"
steps = []
"#;

    let scenario: Scenario = toml::from_str(toml).expect("parse TOML");
    assert!((scenario.config.temperature - 0.0).abs() < f64::EPSILON);
    assert_eq!(scenario.config.thinking, Some(false));
}

#[test]
fn test_custom_assertion_definition() {
    let toml = r#"
[[definitions]]
name = "my_check"
system = "You are a friendly QA bot."
user_template = "Does the page mention {expected_text}?"

[[test]]
name = "dummy"
steps = []
"#;

    let scenario: Scenario = toml::from_str(toml).expect("parse TOML");
    let def = &scenario.definitions[0];
    assert_eq!(def.name, "my_check");
    assert_eq!(def.system.as_deref(), Some("You are a friendly QA bot."));
    assert_eq!(
        def.user_template.as_deref(),
        Some("Does the page mention {expected_text}?")
    );
}

#[test]
fn test_model_params_toml() {
    let toml = r#"
[config]
[config.model_params]
effort = "high"
max_tokens = 2048

[[test]]
name = "dummy"
steps = []
"#;

    let scenario: Scenario = toml::from_str(toml).expect("parse TOML");
    let effort = scenario.config.model_params.get("effort").unwrap();
    assert_eq!(effort.as_str(), Some("high"));
    let max_t = scenario.config.model_params.get("max_tokens").unwrap();
    assert_eq!(max_t.as_i64(), Some(2048));
}