1use futures::stream;
2use roder_api::catalog::{PROVIDER_MOCK, models_for_provider};
3use roder_api::extension::InferenceEngineId;
4use roder_api::inference::*;
5use roder_api::transcript::TranscriptItem;
6
7mod tbench_diagnostics;
8
9pub struct FakeInferenceEngine;
10
11#[async_trait::async_trait]
12impl InferenceEngine for FakeInferenceEngine {
13 fn id(&self) -> InferenceEngineId {
14 PROVIDER_MOCK.to_string()
15 }
16
17 fn capabilities(&self) -> InferenceCapabilities {
18 InferenceCapabilities::text_only()
19 }
20
21 async fn list_models(
22 &self,
23 _ctx: InferenceProviderContext<'_>,
24 ) -> anyhow::Result<Vec<ModelDescriptor>> {
25 Ok(models_for_provider(PROVIDER_MOCK, true))
26 }
27
28 async fn stream_turn(
29 &self,
30 _ctx: InferenceTurnContext<'_>,
31 request: AgentInferenceRequest,
32 ) -> anyhow::Result<InferenceEventStream> {
33 if should_emit_review(&request) {
36 let stream = stream::iter(vec![
37 Ok(InferenceEvent::MessageDelta(MessageDelta {
38 text: FAKE_REVIEW_OUTPUT.to_string(),
39 phase: None,
40 })),
41 Ok(InferenceEvent::Completed(CompletionMetadata {
42 stop_reason: Some("stop".to_string()),
43 provider_response_id: None,
44 })),
45 ]);
46 return Ok(Box::pin(stream));
47 }
48 if should_request_user_input(&request) {
49 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
50 ToolCallCompleted {
51 id: "fake-user-input".to_string(),
52 name: "request_user_input".to_string(),
53 arguments: serde_json::json!({
54 "questions": [{
55 "header": "Choice",
56 "id": "choice",
57 "question": "Which option should be used?",
58 "options": [
59 { "label": "A", "description": "Use option A." },
60 { "label": "B", "description": "Use option B." }
61 ]
62 }]
63 })
64 .to_string(),
65 },
66 ))]);
67 return Ok(Box::pin(stream));
68 }
69 if should_call_external_tool(&request) {
70 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
71 ToolCallCompleted {
72 id: "fake-external-tool".to_string(),
73 name: "acme_lookup".to_string(),
74 arguments: serde_json::json!({ "query": "thread status" }).to_string(),
75 },
76 ))]);
77 return Ok(Box::pin(stream));
78 }
79 if should_update_task_ledger(&request) {
80 let complete = prompt_contains(&request, "FAKE_TASK_LEDGER_COMPLETE");
81 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
82 ToolCallCompleted {
83 id: "fake-task-ledger".to_string(),
84 name: "task_ledger.update".to_string(),
85 arguments: task_ledger_arguments(complete),
86 },
87 ))]);
88 return Ok(Box::pin(stream));
89 }
90 if let Some(tool_call) = tbench_diagnostics::next_tool_call(&request) {
91 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(tool_call))]);
92 return Ok(Box::pin(stream));
93 }
94 if should_write_file(&request) {
95 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
96 ToolCallCompleted {
97 id: "fake-write-file".to_string(),
98 name: "write_file".to_string(),
99 arguments: serde_json::json!({
100 "path": "src/lib.rs",
101 "content": "pub fn fake() -> &'static str { \"verified\" }\n"
102 })
103 .to_string(),
104 },
105 ))]);
106 return Ok(Box::pin(stream));
107 }
108 if should_grep(&request) {
109 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
110 ToolCallCompleted {
111 id: "fake-grep".to_string(),
112 name: "grep".to_string(),
113 arguments: serde_json::json!({
114 "query": "BUG_ROOT_CAUSE_TOKEN",
115 "path": ".",
116 "mode": "indexed",
117 "limit": 20
118 })
119 .to_string(),
120 },
121 ))]);
122 return Ok(Box::pin(stream));
123 }
124 if should_zerolang_graph_dump(&request) {
125 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
126 ToolCallCompleted {
127 id: "fake-zerolang-graph-dump".to_string(),
128 name: "zerolang_graph_dump".to_string(),
129 arguments: serde_json::json!({
130 "input": "src/main.0"
131 })
132 .to_string(),
133 },
134 ))]);
135 return Ok(Box::pin(stream));
136 }
137 if should_zerolang_edit(&request) {
138 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
139 ToolCallCompleted {
140 id: "fake-zerolang-edit".to_string(),
141 name: "zerolang_edit".to_string(),
142 arguments: serde_json::json!({
143 "input": "src/main.0",
144 "graphHash": "graph:f76987e99677f1b3",
145 "operations": [{
146 "op": "set",
147 "node": "#610c78bf",
148 "field": "value",
149 "expect": "hello from zero\n",
150 "value": "hello from roder\n"
151 }],
152 "validate": true
153 })
154 .to_string(),
155 },
156 ))]);
157 return Ok(Box::pin(stream));
158 }
159 if should_discovery_read(&request) {
160 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
161 ToolCallCompleted {
162 id: "fake-discovery-read".to_string(),
163 name: "discovery.read".to_string(),
164 arguments: serde_json::json!({
165 "item_id": "tool:builtin-coding-tools/grep",
166 "promote": true,
167 "limit": 20
168 })
169 .to_string(),
170 },
171 ))]);
172 return Ok(Box::pin(stream));
173 }
174 if should_discovery_search(&request) {
175 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
176 ToolCallCompleted {
177 id: "fake-discovery-search".to_string(),
178 name: "discovery.search".to_string(),
179 arguments: serde_json::json!({
180 "query": "grep",
181 "limit": 20
182 })
183 .to_string(),
184 },
185 ))]);
186 return Ok(Box::pin(stream));
187 }
188 if should_spawn_fake_agent(&request) {
189 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
190 ToolCallCompleted {
191 id: "fake-spawn-agent".to_string(),
192 name: "spawn_agent".to_string(),
193 arguments: serde_json::json!({
194 "task_name": "reviewer",
195 "message": "review the fake agent control smoke"
196 })
197 .to_string(),
198 },
199 ))]);
200 return Ok(Box::pin(stream));
201 }
202 if should_list_fake_agents(&request) {
203 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
204 ToolCallCompleted {
205 id: "fake-list-agents".to_string(),
206 name: "list_agents".to_string(),
207 arguments: "{}".to_string(),
208 },
209 ))]);
210 return Ok(Box::pin(stream));
211 }
212 if should_message_fake_agent(&request) {
213 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
214 ToolCallCompleted {
215 id: "fake-send-message".to_string(),
216 name: "send_message".to_string(),
217 arguments: serde_json::json!({
218 "target": "reviewer",
219 "message": "add one more fake smoke detail"
220 })
221 .to_string(),
222 },
223 ))]);
224 return Ok(Box::pin(stream));
225 }
226 if should_wait_fake_agent(&request) {
227 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
228 ToolCallCompleted {
229 id: "fake-wait-agent".to_string(),
230 name: "wait_agent".to_string(),
231 arguments: serde_json::json!({
232 "target": "reviewer",
233 "timeout_ms": 1000
234 })
235 .to_string(),
236 },
237 ))]);
238 return Ok(Box::pin(stream));
239 }
240 if should_close_fake_agent(&request) {
241 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
242 ToolCallCompleted {
243 id: "fake-close-agent".to_string(),
244 name: "close_agent".to_string(),
245 arguments: serde_json::json!({
246 "target": "reviewer"
247 })
248 .to_string(),
249 },
250 ))]);
251 return Ok(Box::pin(stream));
252 }
253 if should_complete_verification(&request) {
254 let failed = prompt_contains(&request, "FAKE_VERIFICATION_FAILED");
255 let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
256 ToolCallCompleted {
257 id: "fake-verification".to_string(),
258 name: "verification_review".to_string(),
259 arguments: verification_arguments(failed),
260 },
261 ))]);
262 return Ok(Box::pin(stream));
263 }
264 if should_summarize_compaction(&request) {
265 let summary = if prompt_contains(&request, "Review the state snapshot") {
266 "<state_snapshot>goal: continue work\nprogress: fake compaction summary\nnext_steps: proceed</state_snapshot>"
267 } else {
268 "<state_snapshot>goal: continue work\nprogress: fake compaction summary\nnext_steps: proceed</state_snapshot>"
269 };
270 let stream = stream::iter(vec![
271 Ok(InferenceEvent::MessageDelta(MessageDelta {
272 text: summary.to_string(),
273 phase: None,
274 })),
275 Ok(InferenceEvent::Completed(CompletionMetadata {
276 stop_reason: Some("stop".to_string()),
277 provider_response_id: None,
278 })),
279 ]);
280 return Ok(Box::pin(stream));
281 }
282 if verification_failed(&request) {
283 let stream = stream::iter(vec![Ok(InferenceEvent::Failed(InferenceFailure {
284 message: "verification gaps remain: tests not run".to_string(),
285 }))]);
286 return Ok(Box::pin(stream));
287 }
288 if user_input_unavailable(&request) {
289 let stream = stream::iter(vec![Ok(InferenceEvent::Failed(InferenceFailure {
290 message: "clarification unavailable in non-interactive runtime profile".to_string(),
291 }))]);
292 return Ok(Box::pin(stream));
293 }
294 let stream = stream::iter(vec![
295 Ok(InferenceEvent::MessageDelta(MessageDelta {
296 text: "hello".to_string(),
297 phase: None,
298 })),
299 Ok(InferenceEvent::MessageDelta(MessageDelta {
300 text: " from".to_string(),
301 phase: None,
302 })),
303 Ok(InferenceEvent::MessageDelta(MessageDelta {
304 text: " roder".to_string(),
305 phase: None,
306 })),
307 Ok(InferenceEvent::Completed(CompletionMetadata {
308 stop_reason: Some("stop".to_string()),
309 provider_response_id: None,
310 })),
311 ]);
312
313 Ok(Box::pin(stream))
314 }
315}
316
317const FAKE_REVIEW_OUTPUT: &str = r#"{
320 "findings": [
321 {
322 "title": "[P1] Off-by-one in the loop bound",
323 "body": "The loop overruns the slice.",
324 "confidenceScore": 0.8,
325 "priority": "p1",
326 "codeLocation": {
327 "absoluteFilePath": "/tmp/roder-review/src/lib.rs",
328 "lineRange": { "start": 10, "end": 12 }
329 }
330 }
331 ],
332 "overallCorrectness": "patch is incorrect",
333 "overallExplanation": "One blocking bug.",
334 "overallConfidenceScore": 0.7
335}"#;
336
337fn should_emit_review(request: &AgentInferenceRequest) -> bool {
338 prompt_contains(request, "FAKE_REVIEW_FINDINGS")
339}
340
341fn should_request_user_input(request: &AgentInferenceRequest) -> bool {
342 prompt_contains(request, "FAKE_REQUEST_USER_INPUT")
343 && !request.transcript.iter().any(|item| {
344 matches!(
345 item,
346 TranscriptItem::ToolResult(result)
347 if result.name.as_deref() == Some("request_user_input")
348 )
349 })
350}
351
352fn user_input_unavailable(request: &AgentInferenceRequest) -> bool {
353 request.transcript.iter().any(|item| {
354 matches!(
355 item,
356 TranscriptItem::ToolResult(result)
357 if result.name.as_deref() == Some("request_user_input")
358 && result.is_error
359 && result.result.contains("User input is unavailable")
360 )
361 })
362}
363
364fn should_call_external_tool(request: &AgentInferenceRequest) -> bool {
365 prompt_contains(request, "FAKE_EXTERNAL_TOOL") && !has_tool_result(request, "acme_lookup")
366}
367
368fn should_update_task_ledger(request: &AgentInferenceRequest) -> bool {
369 (prompt_contains(request, "FAKE_TASK_LEDGER_UPDATE")
370 || prompt_contains(request, "FAKE_TASK_LEDGER_COMPLETE"))
371 && !request.transcript.iter().any(|item| {
372 matches!(
373 item,
374 TranscriptItem::ToolResult(result)
375 if result.name.as_deref() == Some("task_ledger.update")
376 )
377 })
378}
379
380fn should_write_file(request: &AgentInferenceRequest) -> bool {
381 prompt_contains(request, "FAKE_WRITE_FILE")
382 && !request.transcript.iter().any(|item| {
383 matches!(
384 item,
385 TranscriptItem::ToolResult(result)
386 if result.name.as_deref() == Some("write_file")
387 )
388 })
389}
390
391fn should_grep(request: &AgentInferenceRequest) -> bool {
392 prompt_contains(request, "FAKE_GREP_INDEXED")
393 && !request.transcript.iter().any(|item| {
394 matches!(
395 item,
396 TranscriptItem::ToolResult(result) if result.name.as_deref() == Some("grep")
397 )
398 })
399}
400
401fn should_zerolang_graph_dump(request: &AgentInferenceRequest) -> bool {
402 prompt_contains(request, "FAKE_ZEROLANG_GRAPH_EDIT")
403 && !has_tool_result(request, "zerolang_graph_dump")
404}
405
406fn should_zerolang_edit(request: &AgentInferenceRequest) -> bool {
407 prompt_contains(request, "FAKE_ZEROLANG_GRAPH_EDIT")
408 && has_tool_result(request, "zerolang_graph_dump")
409 && !has_tool_result(request, "zerolang_edit")
410}
411
412fn should_discovery_search(request: &AgentInferenceRequest) -> bool {
413 prompt_contains(request, "FAKE_DISCOVERY_SEARCH")
414 && !request.transcript.iter().any(|item| {
415 matches!(
416 item,
417 TranscriptItem::ToolResult(result)
418 if result.name.as_deref() == Some("discovery.search")
419 )
420 })
421}
422
423fn should_discovery_read(request: &AgentInferenceRequest) -> bool {
424 prompt_contains(request, "FAKE_DISCOVERY_PROMOTE")
425 && request.transcript.iter().any(|item| {
426 matches!(
427 item,
428 TranscriptItem::ToolResult(result)
429 if result.name.as_deref() == Some("discovery.search")
430 )
431 })
432 && !request.transcript.iter().any(|item| {
433 matches!(
434 item,
435 TranscriptItem::ToolResult(result)
436 if result.name.as_deref() == Some("discovery.read")
437 )
438 })
439}
440
441fn should_spawn_fake_agent(request: &AgentInferenceRequest) -> bool {
442 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE") && !has_tool_result(request, "spawn_agent")
443}
444
445fn should_list_fake_agents(request: &AgentInferenceRequest) -> bool {
446 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
447 && has_tool_result(request, "spawn_agent")
448 && !has_tool_result(request, "list_agents")
449}
450
451fn should_message_fake_agent(request: &AgentInferenceRequest) -> bool {
452 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
453 && has_tool_result(request, "list_agents")
454 && !has_tool_result(request, "send_message")
455}
456
457fn should_wait_fake_agent(request: &AgentInferenceRequest) -> bool {
458 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
459 && has_tool_result(request, "send_message")
460 && !has_tool_result(request, "wait_agent")
461}
462
463fn should_close_fake_agent(request: &AgentInferenceRequest) -> bool {
464 prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
465 && has_tool_result(request, "wait_agent")
466 && !has_tool_result(request, "close_agent")
467}
468
469fn should_summarize_compaction(request: &AgentInferenceRequest) -> bool {
470 request.metadata.get("roderCompactionSummary") == Some(&serde_json::json!(true))
471 || prompt_contains(request, crate::compaction::COMPACTION_SUMMARY_PROMPT_MARKER)
472}
473
474fn should_complete_verification(request: &AgentInferenceRequest) -> bool {
475 request.transcript.iter().any(|item| {
476 matches!(
477 item,
478 TranscriptItem::UserMessage(message)
479 if message.text.contains("Verification gate blocked final completion")
480 )
481 }) && !request.transcript.iter().any(|item| {
482 matches!(
483 item,
484 TranscriptItem::ToolResult(result)
485 if result.name.as_deref() == Some("verification_review")
486 )
487 })
488}
489
490fn verification_failed(request: &AgentInferenceRequest) -> bool {
491 request.transcript.iter().any(|item| {
492 matches!(
493 item,
494 TranscriptItem::ToolResult(result)
495 if result.name.as_deref() == Some("verification_review")
496 && result.result.contains("Verification failed")
497 )
498 })
499}
500
501fn has_tool_result(request: &AgentInferenceRequest, name: &str) -> bool {
502 request.transcript.iter().any(|item| {
503 matches!(
504 item,
505 TranscriptItem::ToolResult(result) if result.name.as_deref() == Some(name)
506 )
507 })
508}
509
510fn prompt_contains(request: &AgentInferenceRequest, needle: &str) -> bool {
511 request.transcript.iter().any(|item| {
512 matches!(
513 item,
514 TranscriptItem::UserMessage(message) if message.text.contains(needle)
515 )
516 })
517}
518
519fn task_ledger_arguments(complete: bool) -> String {
520 let second_status = if complete { "completed" } else { "in_progress" };
521 let mut second = serde_json::json!({
522 "id": "verify",
523 "content": "Verify task",
524 "status": second_status
525 });
526 if complete {
527 second["evidence"] = serde_json::json!("fake-provider");
528 }
529 serde_json::json!({
530 "tasks": [
531 { "id": "inspect", "content": "Inspect task", "status": "completed", "evidence": "fake-provider" },
532 second
533 ],
534 "requireCompletionEvidence": true
535 })
536 .to_string()
537}
538
539fn verification_arguments(failed: bool) -> String {
540 let (status, open_gaps) = if failed {
541 ("failed", serde_json::json!(["tests not run"]))
542 } else {
543 ("completed", serde_json::json!([]))
544 };
545 serde_json::json!({
546 "originalTask": "fake verification eval",
547 "changedFiles": ["src/lib.rs"],
548 "toolEvidence": ["write_file wrote src/lib.rs"],
549 "testsRun": if failed { serde_json::json!([]) } else { serde_json::json!(["cargo test -p roder-evals verification"]) },
550 "openGaps": open_gaps,
551 "status": status
552 })
553 .to_string()
554}