1use futures::stream;
2use roder_api::catalog::{PROVIDER_MOCK, models_for_provider};
3use roder_api::extension::InferenceEngineId;
4use roder_api::inference::*;
5use roder_api::transcript::TranscriptItem;
6
7mod tbench_diagnostics;
8
9pub struct FakeInferenceEngine;
10
11#[async_trait::async_trait]
12impl InferenceEngine for FakeInferenceEngine {
13 fn id(&self) -> InferenceEngineId {
14 PROVIDER_MOCK.to_string()
15 }
16
17 fn capabilities(&self) -> InferenceCapabilities {
18 InferenceCapabilities::text_only()
19 }
20
21 async fn list_models(
22 &self,
23 _ctx: InferenceProviderContext<'_>,
24 ) -> anyhow::Result<Vec<ModelDescriptor>> {
25 Ok(models_for_provider(PROVIDER_MOCK, true))
26 }
27
28 async fn stream_turn(
29 &self,
30 _ctx: InferenceTurnContext<'_>,
31 request: AgentInferenceRequest,
32 ) -> anyhow::Result<InferenceEventStream> {
33 if should_request_user_input(&request) {
34 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
35 ToolCallCompleted {
36 id: "fake-user-input".to_string(),
37 name: "request_user_input".to_string(),
38 arguments: serde_json::json!({
39 "questions": [{
40 "header": "Choice",
41 "id": "choice",
42 "question": "Which option should be used?",
43 "options": [
44 { "label": "A", "description": "Use option A." },
45 { "label": "B", "description": "Use option B." }
46 ]
47 }]
48 })
49 .to_string(),
50 },
51 ))]);
52 return Ok(Box::pin(stream));
53 }
54 if should_call_external_tool(&request) {
55 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
56 ToolCallCompleted {
57 id: "fake-external-tool".to_string(),
58 name: "acme_lookup".to_string(),
59 arguments: serde_json::json!({ "query": "thread status" }).to_string(),
60 },
61 ))]);
62 return Ok(Box::pin(stream));
63 }
64 if should_update_task_ledger(&request) {
65 let complete = prompt_contains(&request, "FAKE_TASK_LEDGER_COMPLETE");
66 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
67 ToolCallCompleted {
68 id: "fake-task-ledger".to_string(),
69 name: "task_ledger.update".to_string(),
70 arguments: task_ledger_arguments(complete),
71 },
72 ))]);
73 return Ok(Box::pin(stream));
74 }
75 if let Some(tool_call) = tbench_diagnostics::next_tool_call(&request) {
76 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(tool_call))]);
77 return Ok(Box::pin(stream));
78 }
79 if should_write_file(&request) {
80 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
81 ToolCallCompleted {
82 id: "fake-write-file".to_string(),
83 name: "write_file".to_string(),
84 arguments: serde_json::json!({
85 "path": "src/lib.rs",
86 "content": "pub fn fake() -> &'static str { \"verified\" }\n"
87 })
88 .to_string(),
89 },
90 ))]);
91 return Ok(Box::pin(stream));
92 }
93 if should_grep(&request) {
94 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
95 ToolCallCompleted {
96 id: "fake-grep".to_string(),
97 name: "grep".to_string(),
98 arguments: serde_json::json!({
99 "query": "BUG_ROOT_CAUSE_TOKEN",
100 "path": ".",
101 "mode": "indexed",
102 "limit": 20
103 })
104 .to_string(),
105 },
106 ))]);
107 return Ok(Box::pin(stream));
108 }
109 if should_zerolang_graph_dump(&request) {
110 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
111 ToolCallCompleted {
112 id: "fake-zerolang-graph-dump".to_string(),
113 name: "zerolang_graph_dump".to_string(),
114 arguments: serde_json::json!({
115 "input": "src/main.0"
116 })
117 .to_string(),
118 },
119 ))]);
120 return Ok(Box::pin(stream));
121 }
122 if should_zerolang_edit(&request) {
123 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
124 ToolCallCompleted {
125 id: "fake-zerolang-edit".to_string(),
126 name: "zerolang_edit".to_string(),
127 arguments: serde_json::json!({
128 "input": "src/main.0",
129 "graphHash": "graph:f76987e99677f1b3",
130 "operations": [{
131 "op": "set",
132 "node": "#610c78bf",
133 "field": "value",
134 "expect": "hello from zero\n",
135 "value": "hello from roder\n"
136 }],
137 "validate": true
138 })
139 .to_string(),
140 },
141 ))]);
142 return Ok(Box::pin(stream));
143 }
144 if should_discovery_read(&request) {
145 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
146 ToolCallCompleted {
147 id: "fake-discovery-read".to_string(),
148 name: "discovery.read".to_string(),
149 arguments: serde_json::json!({
150 "item_id": "tool:builtin-coding-tools/grep",
151 "promote": true,
152 "limit": 20
153 })
154 .to_string(),
155 },
156 ))]);
157 return Ok(Box::pin(stream));
158 }
159 if should_discovery_search(&request) {
160 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
161 ToolCallCompleted {
162 id: "fake-discovery-search".to_string(),
163 name: "discovery.search".to_string(),
164 arguments: serde_json::json!({
165 "query": "grep",
166 "limit": 20
167 })
168 .to_string(),
169 },
170 ))]);
171 return Ok(Box::pin(stream));
172 }
173 if should_spawn_fake_agent(&request) {
174 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
175 ToolCallCompleted {
176 id: "fake-spawn-agent".to_string(),
177 name: "spawn_agent".to_string(),
178 arguments: serde_json::json!({
179 "task_name": "reviewer",
180 "message": "review the fake agent control smoke"
181 })
182 .to_string(),
183 },
184 ))]);
185 return Ok(Box::pin(stream));
186 }
187 if should_list_fake_agents(&request) {
188 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
189 ToolCallCompleted {
190 id: "fake-list-agents".to_string(),
191 name: "list_agents".to_string(),
192 arguments: "{}".to_string(),
193 },
194 ))]);
195 return Ok(Box::pin(stream));
196 }
197 if should_message_fake_agent(&request) {
198 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
199 ToolCallCompleted {
200 id: "fake-send-message".to_string(),
201 name: "send_message".to_string(),
202 arguments: serde_json::json!({
203 "target": "reviewer",
204 "message": "add one more fake smoke detail"
205 })
206 .to_string(),
207 },
208 ))]);
209 return Ok(Box::pin(stream));
210 }
211 if should_wait_fake_agent(&request) {
212 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
213 ToolCallCompleted {
214 id: "fake-wait-agent".to_string(),
215 name: "wait_agent".to_string(),
216 arguments: serde_json::json!({
217 "target": "reviewer",
218 "timeout_ms": 1000
219 })
220 .to_string(),
221 },
222 ))]);
223 return Ok(Box::pin(stream));
224 }
225 if should_close_fake_agent(&request) {
226 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
227 ToolCallCompleted {
228 id: "fake-close-agent".to_string(),
229 name: "close_agent".to_string(),
230 arguments: serde_json::json!({
231 "target": "reviewer"
232 })
233 .to_string(),
234 },
235 ))]);
236 return Ok(Box::pin(stream));
237 }
238 if should_complete_verification(&request) {
239 let failed = prompt_contains(&request, "FAKE_VERIFICATION_FAILED");
240 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
241 ToolCallCompleted {
242 id: "fake-verification".to_string(),
243 name: "verification_review".to_string(),
244 arguments: verification_arguments(failed),
245 },
246 ))]);
247 return Ok(Box::pin(stream));
248 }
249 if should_summarize_compaction(&request) {
250 let summary = if prompt_contains(&request, "Review the state snapshot") {
251 "<state_snapshot>goal: continue work\nprogress: fake compaction summary\nnext_steps: proceed</state_snapshot>"
252 } else {
253 "<state_snapshot>goal: continue work\nprogress: fake compaction summary\nnext_steps: proceed</state_snapshot>"
254 };
255 let stream = stream::iter(vec![
256 Ok(InferenceEvent::MessageDelta(MessageDelta {
257 text: summary.to_string(),
258 phase: None,
259 })),
260 Ok(InferenceEvent::Completed(CompletionMetadata {
261 stop_reason: Some("stop".to_string()),
262 provider_response_id: None,
263 })),
264 ]);
265 return Ok(Box::pin(stream));
266 }
267 if verification_failed(&request) {
268 let stream = stream::iter(vec![Ok(InferenceEvent::Failed(InferenceFailure {
269 message: "verification gaps remain: tests not run".to_string(),
270 }))]);
271 return Ok(Box::pin(stream));
272 }
273 if user_input_unavailable(&request) {
274 let stream = stream::iter(vec![Ok(InferenceEvent::Failed(InferenceFailure {
275 message: "clarification unavailable in non-interactive runtime profile".to_string(),
276 }))]);
277 return Ok(Box::pin(stream));
278 }
279 let stream = stream::iter(vec![
280 Ok(InferenceEvent::MessageDelta(MessageDelta {
281 text: "hello".to_string(),
282 phase: None,
283 })),
284 Ok(InferenceEvent::MessageDelta(MessageDelta {
285 text: " from".to_string(),
286 phase: None,
287 })),
288 Ok(InferenceEvent::MessageDelta(MessageDelta {
289 text: " roder".to_string(),
290 phase: None,
291 })),
292 Ok(InferenceEvent::Completed(CompletionMetadata {
293 stop_reason: Some("stop".to_string()),
294 provider_response_id: None,
295 })),
296 ]);
297
298 Ok(Box::pin(stream))
299 }
300}
301
302fn should_request_user_input(request: &AgentInferenceRequest) -> bool {
303 prompt_contains(request, "FAKE_REQUEST_USER_INPUT")
304 && !request.transcript.iter().any(|item| {
305 matches!(
306 item,
307 TranscriptItem::ToolResult(result)
308 if result.name.as_deref() == Some("request_user_input")
309 )
310 })
311}
312
313fn user_input_unavailable(request: &AgentInferenceRequest) -> bool {
314 request.transcript.iter().any(|item| {
315 matches!(
316 item,
317 TranscriptItem::ToolResult(result)
318 if result.name.as_deref() == Some("request_user_input")
319 && result.is_error
320 && result.result.contains("User input is unavailable")
321 )
322 })
323}
324
325fn should_call_external_tool(request: &AgentInferenceRequest) -> bool {
326 prompt_contains(request, "FAKE_EXTERNAL_TOOL") && !has_tool_result(request, "acme_lookup")
327}
328
329fn should_update_task_ledger(request: &AgentInferenceRequest) -> bool {
330 (prompt_contains(request, "FAKE_TASK_LEDGER_UPDATE")
331 || prompt_contains(request, "FAKE_TASK_LEDGER_COMPLETE"))
332 && !request.transcript.iter().any(|item| {
333 matches!(
334 item,
335 TranscriptItem::ToolResult(result)
336 if result.name.as_deref() == Some("task_ledger.update")
337 )
338 })
339}
340
341fn should_write_file(request: &AgentInferenceRequest) -> bool {
342 prompt_contains(request, "FAKE_WRITE_FILE")
343 && !request.transcript.iter().any(|item| {
344 matches!(
345 item,
346 TranscriptItem::ToolResult(result)
347 if result.name.as_deref() == Some("write_file")
348 )
349 })
350}
351
352fn should_grep(request: &AgentInferenceRequest) -> bool {
353 prompt_contains(request, "FAKE_GREP_INDEXED")
354 && !request.transcript.iter().any(|item| {
355 matches!(
356 item,
357 TranscriptItem::ToolResult(result) if result.name.as_deref() == Some("grep")
358 )
359 })
360}
361
362fn should_zerolang_graph_dump(request: &AgentInferenceRequest) -> bool {
363 prompt_contains(request, "FAKE_ZEROLANG_GRAPH_EDIT")
364 && !has_tool_result(request, "zerolang_graph_dump")
365}
366
367fn should_zerolang_edit(request: &AgentInferenceRequest) -> bool {
368 prompt_contains(request, "FAKE_ZEROLANG_GRAPH_EDIT")
369 && has_tool_result(request, "zerolang_graph_dump")
370 && !has_tool_result(request, "zerolang_edit")
371}
372
373fn should_discovery_search(request: &AgentInferenceRequest) -> bool {
374 prompt_contains(request, "FAKE_DISCOVERY_SEARCH")
375 && !request.transcript.iter().any(|item| {
376 matches!(
377 item,
378 TranscriptItem::ToolResult(result)
379 if result.name.as_deref() == Some("discovery.search")
380 )
381 })
382}
383
384fn should_discovery_read(request: &AgentInferenceRequest) -> bool {
385 prompt_contains(request, "FAKE_DISCOVERY_PROMOTE")
386 && request.transcript.iter().any(|item| {
387 matches!(
388 item,
389 TranscriptItem::ToolResult(result)
390 if result.name.as_deref() == Some("discovery.search")
391 )
392 })
393 && !request.transcript.iter().any(|item| {
394 matches!(
395 item,
396 TranscriptItem::ToolResult(result)
397 if result.name.as_deref() == Some("discovery.read")
398 )
399 })
400}
401
402fn should_spawn_fake_agent(request: &AgentInferenceRequest) -> bool {
403 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE") && !has_tool_result(request, "spawn_agent")
404}
405
406fn should_list_fake_agents(request: &AgentInferenceRequest) -> bool {
407 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
408 && has_tool_result(request, "spawn_agent")
409 && !has_tool_result(request, "list_agents")
410}
411
412fn should_message_fake_agent(request: &AgentInferenceRequest) -> bool {
413 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
414 && has_tool_result(request, "list_agents")
415 && !has_tool_result(request, "send_message")
416}
417
418fn should_wait_fake_agent(request: &AgentInferenceRequest) -> bool {
419 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
420 && has_tool_result(request, "send_message")
421 && !has_tool_result(request, "wait_agent")
422}
423
424fn should_close_fake_agent(request: &AgentInferenceRequest) -> bool {
425 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
426 && has_tool_result(request, "wait_agent")
427 && !has_tool_result(request, "close_agent")
428}
429
430fn should_summarize_compaction(request: &AgentInferenceRequest) -> bool {
431 request.metadata.get("roderCompactionSummary") == Some(&serde_json::json!(true))
432 || prompt_contains(request, crate::compaction::COMPACTION_SUMMARY_PROMPT_MARKER)
433}
434
435fn should_complete_verification(request: &AgentInferenceRequest) -> bool {
436 request.transcript.iter().any(|item| {
437 matches!(
438 item,
439 TranscriptItem::UserMessage(message)
440 if message.text.contains("Verification gate blocked final completion")
441 )
442 }) && !request.transcript.iter().any(|item| {
443 matches!(
444 item,
445 TranscriptItem::ToolResult(result)
446 if result.name.as_deref() == Some("verification_review")
447 )
448 })
449}
450
451fn verification_failed(request: &AgentInferenceRequest) -> bool {
452 request.transcript.iter().any(|item| {
453 matches!(
454 item,
455 TranscriptItem::ToolResult(result)
456 if result.name.as_deref() == Some("verification_review")
457 && result.result.contains("Verification failed")
458 )
459 })
460}
461
462fn has_tool_result(request: &AgentInferenceRequest, name: &str) -> bool {
463 request.transcript.iter().any(|item| {
464 matches!(
465 item,
466 TranscriptItem::ToolResult(result) if result.name.as_deref() == Some(name)
467 )
468 })
469}
470
471fn prompt_contains(request: &AgentInferenceRequest, needle: &str) -> bool {
472 request.transcript.iter().any(|item| {
473 matches!(
474 item,
475 TranscriptItem::UserMessage(message) if message.text.contains(needle)
476 )
477 })
478}
479
480fn task_ledger_arguments(complete: bool) -> String {
481 let second_status = if complete { "completed" } else { "in_progress" };
482 let mut second = serde_json::json!({
483 "id": "verify",
484 "content": "Verify task",
485 "status": second_status
486 });
487 if complete {
488 second["evidence"] = serde_json::json!("fake-provider");
489 }
490 serde_json::json!({
491 "tasks": [
492 { "id": "inspect", "content": "Inspect task", "status": "completed", "evidence": "fake-provider" },
493 second
494 ],
495 "requireCompletionEvidence": true
496 })
497 .to_string()
498}
499
500fn verification_arguments(failed: bool) -> String {
501 let (status, open_gaps) = if failed {
502 ("failed", serde_json::json!(["tests not run"]))
503 } else {
504 ("completed", serde_json::json!([]))
505 };
506 serde_json::json!({
507 "originalTask": "fake verification eval",
508 "changedFiles": ["src/lib.rs"],
509 "toolEvidence": ["write_file wrote src/lib.rs"],
510 "testsRun": if failed { serde_json::json!([]) } else { serde_json::json!(["cargo test -p roder-evals verification"]) },
511 "openGaps": open_gaps,
512 "status": status
513 })
514 .to_string()
515}