1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
mod common;
use std::fs::{create_dir, write};
use std::path::PathBuf;
use predicates::prelude::*;
use tempfile::{tempdir, TempDir};
use common::slopguard;
/// Build a project dir with a source file, a custom-rules dir holding one
/// `ai_check` rule, and a config file. Returns (project_dir, config_path).
///
/// All rulesets are disabled so the only active rule is the custom AI rule:
/// this isolates the AI phase from builtin AST findings.
fn ai_fixture(ai_enabled: bool) -> (TempDir, PathBuf) {
let proj = tempdir().unwrap();
// A file the AI rule's AST pre-filter would match (a SAFETY comment).
write(
proj.path().join("main.rs"),
"fn main() {\n // SAFETY: trust me\n let _x = 1;\n}\n",
)
.unwrap();
let rules_dir = proj.path().join("custom-rules");
create_dir(&rules_dir).unwrap();
write(
rules_dir.join("ai.yml"),
r#"
id: ai-safety-demo
language: rust
severity: warning
category: slop
message: "AI: SAFETY comment may be meaningless"
rule:
kind: line_comment
regex: 'SAFETY'
ai_check:
prompt: "Is this SAFETY comment meaningful?\n{{code}}"
"#,
)
.unwrap();
let config_path = proj.path().join("slopguard.toml");
// Forward slashes are valid in TOML strings and fine on the test platforms.
let custom_dir = rules_dir.to_str().unwrap().replace('\\', "/");
write(
&config_path,
format!(
"[rulesets]\nslop = false\nsecurity = false\ncorrectness = false\n\n\
[rules]\ncustom_dirs = [\"{custom_dir}\"]\n\n\
[ai]\nenabled = {ai_enabled}\nprovider = \"api\"\nvendor = \"anthropic\"\n"
),
)
.unwrap();
(proj, config_path)
}
/// Like [`ai_fixture`] but enables the System One classifier instead of the
/// LLM. `[ai]` is left disabled so the classifier path is the one under test.
fn classifier_fixture() -> (TempDir, PathBuf) {
let proj = tempdir().unwrap();
write(
proj.path().join("main.rs"),
"fn main() {\n // SAFETY: trust me\n let _x = 1;\n}\n",
)
.unwrap();
let rules_dir = proj.path().join("custom-rules");
create_dir(&rules_dir).unwrap();
write(
rules_dir.join("ai.yml"),
r#"
id: ai-safety-demo
language: rust
severity: warning
category: slop
message: "AI: SAFETY comment may be meaningless"
rule:
kind: line_comment
regex: 'SAFETY'
ai_check:
prompt: "Is this SAFETY comment meaningful?\n{{code}}"
"#,
)
.unwrap();
let config_path = proj.path().join("slopguard.toml");
let custom_dir = rules_dir.to_str().unwrap().replace('\\', "/");
write(
&config_path,
format!(
"[rulesets]\nslop = false\nsecurity = false\ncorrectness = false\n\n\
[rules]\ncustom_dirs = [\"{custom_dir}\"]\n\n\
[ai]\nenabled = false\n\n\
[ai.classifier]\nenabled = true\ntransport = \"direct\"\n"
),
)
.unwrap();
(proj, config_path)
}
#[test]
fn scan_classifier_without_key_warns_and_skips() {
// WHY: the classifier is opt-in strict. With [ai.classifier].enabled = true
// but no TYPESAFE_API_KEY, the scan must fail fast (before any network call)
// with a single warning naming the missing credential, and still succeed.
let (proj, config) = classifier_fixture();
slopguard()
// Ensure a locally-set key does not turn this into a real network call.
.env_remove("TYPESAFE_API_KEY")
.args([
"scan",
"--no-colors",
"--config",
config.to_str().unwrap(),
proj.path().to_str().unwrap(),
])
.assert()
.success()
.stdout(predicate::str::contains("0 errors, 0 warnings"))
.stderr(predicate::str::contains("AI rules skipped"))
// Names the exact missing credential, not a generic message.
.stderr(predicate::str::contains("TYPESAFE_API_KEY"));
}
#[test]
fn scan_no_ai_skips_classifier_too() {
// WHY (gate 5): --no-ai must skip Jev as well, not just the LLM. With the
// classifier enabled but --no-ai passed, ai_check rules are dropped before
// the classifier phase, so there is no call and no warning.
let (proj, config) = classifier_fixture();
slopguard()
.env_remove("TYPESAFE_API_KEY")
.args([
"scan",
"--no-ai",
"--no-colors",
"--config",
config.to_str().unwrap(),
proj.path().to_str().unwrap(),
])
.assert()
.success()
.stdout(predicate::str::contains("0 errors, 0 warnings"))
.stderr(predicate::str::is_empty());
}
#[test]
fn scan_no_ai_makes_no_llm_call_and_is_silent() {
// WHY: --no-ai must skip AI rules entirely without any network call, error,
// or warning. The issue spec requires "pas de warning": the AI subsystem
// must emit nothing on stderr so `scan --no-ai 2>&1 | grep -c AI` is 0.
let (proj, config) = ai_fixture(true);
slopguard()
.args([
"scan",
"--no-ai",
"--no-colors",
"--config",
config.to_str().unwrap(),
proj.path().to_str().unwrap(),
])
.assert()
.success()
// AI rules are excluded from AST results: no finding leaks through.
.stdout(predicate::str::contains("0 errors, 0 warnings"))
// No warning at all: the fixture's custom rule id and message both
// contain "AI", so any leak would show up here.
.stderr(predicate::str::is_empty());
}
#[test]
fn scan_without_provider_warns_and_skips_ai_rules() {
// WHY: with an active AI rule but no usable provider ([ai].enabled = false),
// the scan must emit a single "N AI rules skipped" warning and proceed,
// never calling an LLM and never reporting the unconfirmed candidate.
let (proj, config) = ai_fixture(false);
slopguard()
.args([
"scan",
"--no-colors",
"--config",
config.to_str().unwrap(),
proj.path().to_str().unwrap(),
])
.assert()
.success()
.stdout(predicate::str::contains("0 errors, 0 warnings"))
// A single clear warning naming the reason (AI disabled), not an LLM call.
.stderr(predicate::str::contains("AI rules skipped"))
.stderr(predicate::str::contains("AI is disabled"));
}
#[test]
fn list_shows_ai_rules_with_type_column() {
// WHY: `slopguard list` must surface AI rules and mark them with a type
// (ast vs ai) so users can tell which rules need a provider.
slopguard()
.args(["list"])
.assert()
.success()
.stdout(predicate::str::contains("type"))
.stdout(predicate::str::contains("ai-safety-comment-validation"))
// The builtin AI rule is tagged "ai", not "ast".
.stdout(
predicate::str::is_match(r"ai-safety-comment-validation\s+rust\s+\S+\s+security\s+ai")
.unwrap(),
);
}
#[test]
fn explain_ai_rule_shows_prompt_template() {
// WHY: the issue requires `explain` to show the prompt template for AI
// rules, so a rule author can inspect what is sent to the model.
slopguard()
.args(["explain", "ai-safety-comment-validation"])
.assert()
.success()
.stdout(predicate::str::contains("prompt:"))
.stdout(predicate::str::contains("type: ai"));
}