1use futures::stream;
2use roder_api::catalog::{PROVIDER_MOCK, models_for_provider};
3use roder_api::extension::InferenceEngineId;
4use roder_api::inference::*;
5use roder_api::transcript::TranscriptItem;
6
7mod tbench_diagnostics;
8mod tool_stream;
9
10pub struct FakeInferenceEngine;
11
12#[async_trait::async_trait]
13impl InferenceEngine for FakeInferenceEngine {
14 fn id(&self) -> InferenceEngineId {
15 PROVIDER_MOCK.to_string()
16 }
17
18 fn capabilities(&self) -> InferenceCapabilities {
19 InferenceCapabilities::text_only()
20 }
21
22 async fn list_models(
23 &self,
24 _ctx: InferenceProviderContext<'_>,
25 ) -> anyhow::Result<Vec<ModelDescriptor>> {
26 Ok(models_for_provider(PROVIDER_MOCK, true))
27 }
28
29 async fn stream_turn(
30 &self,
31 _ctx: InferenceTurnContext<'_>,
32 request: AgentInferenceRequest,
33 ) -> anyhow::Result<InferenceEventStream> {
34 if should_emit_review(&request) {
37 let stream = stream::iter(vec![
38 Ok(InferenceEvent::MessageDelta(MessageDelta {
39 text: FAKE_REVIEW_OUTPUT.to_string(),
40 phase: None,
41 })),
42 Ok(InferenceEvent::Completed(CompletionMetadata {
43 stop_reason: Some("stop".to_string()),
44 provider_response_id: None,
45 })),
46 ]);
47 return Ok(Box::pin(stream));
48 }
49 if should_request_user_input(&request) {
50 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
51 ToolCallCompleted {
52 id: "fake-user-input".to_string(),
53 name: "request_user_input".to_string(),
54 arguments: serde_json::json!({
55 "questions": [{
56 "header": "Choice",
57 "id": "choice",
58 "question": "Which option should be used?",
59 "options": [
60 { "label": "A", "description": "Use option A." },
61 { "label": "B", "description": "Use option B." }
62 ]
63 }]
64 })
65 .to_string(),
66 },
67 ))]);
68 return Ok(tool_stream::completed(stream));
69 }
70 if should_call_external_tool(&request) {
71 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
72 ToolCallCompleted {
73 id: "fake-external-tool".to_string(),
74 name: "acme_lookup".to_string(),
75 arguments: serde_json::json!({ "query": "thread status" }).to_string(),
76 },
77 ))]);
78 return Ok(tool_stream::completed(stream));
79 }
80 if should_update_task_ledger(&request) {
81 let complete = prompt_contains(&request, "FAKE_TASK_LEDGER_COMPLETE");
82 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
83 ToolCallCompleted {
84 id: "fake-task-ledger".to_string(),
85 name: "task_ledger.update".to_string(),
86 arguments: task_ledger_arguments(complete),
87 },
88 ))]);
89 return Ok(tool_stream::completed(stream));
90 }
91 if let Some(tool_call) = tbench_diagnostics::next_tool_call(&request) {
92 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(tool_call))]);
93 return Ok(tool_stream::completed(stream));
94 }
95 if should_write_file(&request) {
96 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
97 ToolCallCompleted {
98 id: "fake-write-file".to_string(),
99 name: "write_file".to_string(),
100 arguments: serde_json::json!({
101 "path": "src/lib.rs",
102 "content": "pub fn fake() -> &'static str { \"verified\" }\n"
103 })
104 .to_string(),
105 },
106 ))]);
107 return Ok(tool_stream::completed(stream));
108 }
109 if should_grep(&request) {
110 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
111 ToolCallCompleted {
112 id: "fake-grep".to_string(),
113 name: "grep".to_string(),
114 arguments: serde_json::json!({
115 "query": "BUG_ROOT_CAUSE_TOKEN",
116 "path": ".",
117 "mode": "indexed",
118 "limit": 20
119 })
120 .to_string(),
121 },
122 ))]);
123 return Ok(tool_stream::completed(stream));
124 }
125 if should_zerolang_graph_dump(&request) {
126 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
127 ToolCallCompleted {
128 id: "fake-zerolang-graph-dump".to_string(),
129 name: "zerolang_graph_dump".to_string(),
130 arguments: serde_json::json!({
131 "input": "src/main.0"
132 })
133 .to_string(),
134 },
135 ))]);
136 return Ok(tool_stream::completed(stream));
137 }
138 if should_zerolang_edit(&request) {
139 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
140 ToolCallCompleted {
141 id: "fake-zerolang-edit".to_string(),
142 name: "zerolang_edit".to_string(),
143 arguments: serde_json::json!({
144 "input": "src/main.0",
145 "graphHash": "graph:f76987e99677f1b3",
146 "operations": [{
147 "op": "set",
148 "node": "#610c78bf",
149 "field": "value",
150 "expect": "hello from zero\n",
151 "value": "hello from roder\n"
152 }],
153 "validate": true
154 })
155 .to_string(),
156 },
157 ))]);
158 return Ok(tool_stream::completed(stream));
159 }
160 if should_discovery_read(&request) {
161 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
162 ToolCallCompleted {
163 id: "fake-discovery-read".to_string(),
164 name: "discovery.read".to_string(),
165 arguments: serde_json::json!({
166 "item_id": "tool:builtin-coding-tools/grep",
167 "promote": true,
168 "limit": 20
169 })
170 .to_string(),
171 },
172 ))]);
173 return Ok(tool_stream::completed(stream));
174 }
175 if should_discovery_search(&request) {
176 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
177 ToolCallCompleted {
178 id: "fake-discovery-search".to_string(),
179 name: "discovery.search".to_string(),
180 arguments: serde_json::json!({
181 "query": "grep",
182 "limit": 20
183 })
184 .to_string(),
185 },
186 ))]);
187 return Ok(tool_stream::completed(stream));
188 }
189 if should_spawn_fake_agent(&request) {
190 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
191 ToolCallCompleted {
192 id: "fake-spawn-agent".to_string(),
193 name: "spawn_agent".to_string(),
194 arguments: serde_json::json!({
195 "task_name": "reviewer",
196 "message": "review the fake agent control smoke"
197 })
198 .to_string(),
199 },
200 ))]);
201 return Ok(tool_stream::completed(stream));
202 }
203 if should_list_fake_agents(&request) {
204 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
205 ToolCallCompleted {
206 id: "fake-list-agents".to_string(),
207 name: "list_agents".to_string(),
208 arguments: "{}".to_string(),
209 },
210 ))]);
211 return Ok(tool_stream::completed(stream));
212 }
213 if should_message_fake_agent(&request) {
214 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
215 ToolCallCompleted {
216 id: "fake-send-message".to_string(),
217 name: "send_message".to_string(),
218 arguments: serde_json::json!({
219 "target": "reviewer",
220 "message": "add one more fake smoke detail"
221 })
222 .to_string(),
223 },
224 ))]);
225 return Ok(tool_stream::completed(stream));
226 }
227 if should_wait_fake_agent(&request) {
228 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
229 ToolCallCompleted {
230 id: "fake-wait-agent".to_string(),
231 name: "wait_agent".to_string(),
232 arguments: serde_json::json!({
233 "target": "reviewer",
234 "timeout_ms": 1000
235 })
236 .to_string(),
237 },
238 ))]);
239 return Ok(tool_stream::completed(stream));
240 }
241 if should_close_fake_agent(&request) {
242 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
243 ToolCallCompleted {
244 id: "fake-close-agent".to_string(),
245 name: "close_agent".to_string(),
246 arguments: serde_json::json!({
247 "target": "reviewer"
248 })
249 .to_string(),
250 },
251 ))]);
252 return Ok(tool_stream::completed(stream));
253 }
254 if should_complete_verification(&request) {
255 let failed = prompt_contains(&request, "FAKE_VERIFICATION_FAILED");
256 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
257 ToolCallCompleted {
258 id: "fake-verification".to_string(),
259 name: "verification_review".to_string(),
260 arguments: verification_arguments(failed),
261 },
262 ))]);
263 return Ok(tool_stream::completed(stream));
264 }
265 if should_summarize_compaction(&request) {
266 let summary = if prompt_contains(&request, "Review the state snapshot") {
267 "<state_snapshot>goal: continue work\nprogress: fake compaction summary\nnext_steps: proceed</state_snapshot>"
268 } else {
269 "<state_snapshot>goal: continue work\nprogress: fake compaction summary\nnext_steps: proceed</state_snapshot>"
270 };
271 let stream = stream::iter(vec![
272 Ok(InferenceEvent::MessageDelta(MessageDelta {
273 text: summary.to_string(),
274 phase: None,
275 })),
276 Ok(InferenceEvent::Completed(CompletionMetadata {
277 stop_reason: Some("stop".to_string()),
278 provider_response_id: None,
279 })),
280 ]);
281 return Ok(Box::pin(stream));
282 }
283 if verification_failed(&request) {
284 let stream = stream::iter(vec![Ok(InferenceEvent::Failed(InferenceFailure {
285 message: "verification gaps remain: tests not run".to_string(),
286 }))]);
287 return Ok(Box::pin(stream));
288 }
289 if user_input_unavailable(&request) {
290 let stream = stream::iter(vec![Ok(InferenceEvent::Failed(InferenceFailure {
291 message: "clarification unavailable in non-interactive runtime profile".to_string(),
292 }))]);
293 return Ok(Box::pin(stream));
294 }
295 let stream = stream::iter(vec![
296 Ok(InferenceEvent::MessageDelta(MessageDelta {
297 text: "hello".to_string(),
298 phase: None,
299 })),
300 Ok(InferenceEvent::MessageDelta(MessageDelta {
301 text: " from".to_string(),
302 phase: None,
303 })),
304 Ok(InferenceEvent::MessageDelta(MessageDelta {
305 text: " roder".to_string(),
306 phase: None,
307 })),
308 Ok(InferenceEvent::Completed(CompletionMetadata {
309 stop_reason: Some("stop".to_string()),
310 provider_response_id: None,
311 })),
312 ]);
313
314 Ok(Box::pin(stream))
315 }
316}
317
318const FAKE_REVIEW_OUTPUT: &str = r#"{
321 "findings": [
322 {
323 "title": "[P1] Off-by-one in the loop bound",
324 "body": "The loop overruns the slice.",
325 "confidenceScore": 0.8,
326 "priority": "p1",
327 "codeLocation": {
328 "absoluteFilePath": "/tmp/roder-review/src/lib.rs",
329 "lineRange": { "start": 10, "end": 12 }
330 }
331 }
332 ],
333 "overallCorrectness": "patch is incorrect",
334 "overallExplanation": "One blocking bug.",
335 "overallConfidenceScore": 0.7
336}"#;
337
338fn should_emit_review(request: &AgentInferenceRequest) -> bool {
339 prompt_contains(request, "FAKE_REVIEW_FINDINGS")
340}
341
342fn should_request_user_input(request: &AgentInferenceRequest) -> bool {
343 prompt_contains(request, "FAKE_REQUEST_USER_INPUT")
344 && !request.transcript.iter().any(|item| {
345 matches!(
346 item,
347 TranscriptItem::ToolResult(result)
348 if result.name.as_deref() == Some("request_user_input")
349 )
350 })
351}
352
353fn user_input_unavailable(request: &AgentInferenceRequest) -> bool {
354 request.transcript.iter().any(|item| {
355 matches!(
356 item,
357 TranscriptItem::ToolResult(result)
358 if result.name.as_deref() == Some("request_user_input")
359 && result.is_error
360 && result.result.contains("User input is unavailable")
361 )
362 })
363}
364
365fn should_call_external_tool(request: &AgentInferenceRequest) -> bool {
366 prompt_contains(request, "FAKE_EXTERNAL_TOOL") && !has_tool_result(request, "acme_lookup")
367}
368
369fn should_update_task_ledger(request: &AgentInferenceRequest) -> bool {
370 (prompt_contains(request, "FAKE_TASK_LEDGER_UPDATE")
371 || prompt_contains(request, "FAKE_TASK_LEDGER_COMPLETE"))
372 && !request.transcript.iter().any(|item| {
373 matches!(
374 item,
375 TranscriptItem::ToolResult(result)
376 if result.name.as_deref() == Some("task_ledger.update")
377 )
378 })
379}
380
381fn should_write_file(request: &AgentInferenceRequest) -> bool {
382 prompt_contains(request, "FAKE_WRITE_FILE")
383 && !request.transcript.iter().any(|item| {
384 matches!(
385 item,
386 TranscriptItem::ToolResult(result)
387 if result.name.as_deref() == Some("write_file")
388 )
389 })
390}
391
392fn should_grep(request: &AgentInferenceRequest) -> bool {
393 prompt_contains(request, "FAKE_GREP_INDEXED")
394 && !request.transcript.iter().any(|item| {
395 matches!(
396 item,
397 TranscriptItem::ToolResult(result) if result.name.as_deref() == Some("grep")
398 )
399 })
400}
401
402fn should_zerolang_graph_dump(request: &AgentInferenceRequest) -> bool {
403 prompt_contains(request, "FAKE_ZEROLANG_GRAPH_EDIT")
404 && !has_tool_result(request, "zerolang_graph_dump")
405}
406
407fn should_zerolang_edit(request: &AgentInferenceRequest) -> bool {
408 prompt_contains(request, "FAKE_ZEROLANG_GRAPH_EDIT")
409 && has_tool_result(request, "zerolang_graph_dump")
410 && !has_tool_result(request, "zerolang_edit")
411}
412
413fn should_discovery_search(request: &AgentInferenceRequest) -> bool {
414 prompt_contains(request, "FAKE_DISCOVERY_SEARCH")
415 && !request.transcript.iter().any(|item| {
416 matches!(
417 item,
418 TranscriptItem::ToolResult(result)
419 if result.name.as_deref() == Some("discovery.search")
420 )
421 })
422}
423
424fn should_discovery_read(request: &AgentInferenceRequest) -> bool {
425 prompt_contains(request, "FAKE_DISCOVERY_PROMOTE")
426 && request.transcript.iter().any(|item| {
427 matches!(
428 item,
429 TranscriptItem::ToolResult(result)
430 if result.name.as_deref() == Some("discovery.search")
431 )
432 })
433 && !request.transcript.iter().any(|item| {
434 matches!(
435 item,
436 TranscriptItem::ToolResult(result)
437 if result.name.as_deref() == Some("discovery.read")
438 )
439 })
440}
441
442fn should_spawn_fake_agent(request: &AgentInferenceRequest) -> bool {
443 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE") && !has_tool_result(request, "spawn_agent")
444}
445
446fn should_list_fake_agents(request: &AgentInferenceRequest) -> bool {
447 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
448 && has_tool_result(request, "spawn_agent")
449 && !has_tool_result(request, "list_agents")
450}
451
452fn should_message_fake_agent(request: &AgentInferenceRequest) -> bool {
453 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
454 && has_tool_result(request, "list_agents")
455 && !has_tool_result(request, "send_message")
456}
457
458fn should_wait_fake_agent(request: &AgentInferenceRequest) -> bool {
459 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
460 && has_tool_result(request, "send_message")
461 && !has_tool_result(request, "wait_agent")
462}
463
464fn should_close_fake_agent(request: &AgentInferenceRequest) -> bool {
465 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
466 && has_tool_result(request, "wait_agent")
467 && !has_tool_result(request, "close_agent")
468}
469
470fn should_summarize_compaction(request: &AgentInferenceRequest) -> bool {
471 request.metadata.get("roderCompactionSummary") == Some(&serde_json::json!(true))
472 || prompt_contains(request, crate::compaction::COMPACTION_SUMMARY_PROMPT_MARKER)
473}
474
475fn should_complete_verification(request: &AgentInferenceRequest) -> bool {
476 request.transcript.iter().any(|item| {
477 matches!(
478 item,
479 TranscriptItem::UserMessage(message)
480 if message.text.contains("Verification gate blocked final completion")
481 )
482 }) && !request.transcript.iter().any(|item| {
483 matches!(
484 item,
485 TranscriptItem::ToolResult(result)
486 if result.name.as_deref() == Some("verification_review")
487 )
488 })
489}
490
491fn verification_failed(request: &AgentInferenceRequest) -> bool {
492 request.transcript.iter().any(|item| {
493 matches!(
494 item,
495 TranscriptItem::ToolResult(result)
496 if result.name.as_deref() == Some("verification_review")
497 && result.result.contains("Verification failed")
498 )
499 })
500}
501
502fn has_tool_result(request: &AgentInferenceRequest, name: &str) -> bool {
503 request.transcript.iter().any(|item| {
504 matches!(
505 item,
506 TranscriptItem::ToolResult(result) if result.name.as_deref() == Some(name)
507 )
508 })
509}
510
511fn prompt_contains(request: &AgentInferenceRequest, needle: &str) -> bool {
512 request.transcript.iter().any(|item| {
513 matches!(
514 item,
515 TranscriptItem::UserMessage(message) if message.text.contains(needle)
516 )
517 })
518}
519
520fn task_ledger_arguments(complete: bool) -> String {
521 let second_status = if complete { "completed" } else { "in_progress" };
522 let mut second = serde_json::json!({
523 "id": "verify",
524 "content": "Verify task",
525 "status": second_status
526 });
527 if complete {
528 second["evidence"] = serde_json::json!("fake-provider");
529 }
530 serde_json::json!({
531 "tasks": [
532 { "id": "inspect", "content": "Inspect task", "status": "completed", "evidence": "fake-provider" },
533 second
534 ],
535 "requireCompletionEvidence": true
536 })
537 .to_string()
538}
539
540fn verification_arguments(failed: bool) -> String {
541 let (status, open_gaps) = if failed {
542 ("failed", serde_json::json!(["tests not run"]))
543 } else {
544 ("completed", serde_json::json!([]))
545 };
546 serde_json::json!({
547 "originalTask": "fake verification eval",
548 "changedFiles": ["src/lib.rs"],
549 "toolEvidence": ["write_file wrote src/lib.rs"],
550 "testsRun": if failed { serde_json::json!([]) } else { serde_json::json!(["cargo test -p roder-evals verification"]) },
551 "openGaps": open_gaps,
552 "status": status
553 })
554 .to_string()
555}