roder_core/
fake_provider.rs

1use futures::stream;
2use roder_api::catalog::{PROVIDER_MOCK, models_for_provider};
3use roder_api::extension::InferenceEngineId;
4use roder_api::inference::*;
5use roder_api::transcript::TranscriptItem;
6
7mod tbench_diagnostics;
8mod tool_stream;
9
10pub struct FakeInferenceEngine;
11
12#[async_trait::async_trait]
13impl InferenceEngine for FakeInferenceEngine {
14    fn id(&self) -> InferenceEngineId {
15        PROVIDER_MOCK.to_string()
16    }
17
18    fn capabilities(&self) -> InferenceCapabilities {
19        InferenceCapabilities::text_only()
20    }
21
22    async fn list_models(
23        &self,
24        _ctx: InferenceProviderContext<'_>,
25    ) -> anyhow::Result<Vec<ModelDescriptor>> {
26        Ok(models_for_provider(PROVIDER_MOCK, true))
27    }
28
29    async fn stream_turn(
30        &self,
31        _ctx: InferenceTurnContext<'_>,
32        request: AgentInferenceRequest,
33    ) -> anyhow::Result<InferenceEventStream> {
34        // Checked first: a review turn replaces the system prompt with the
35        // rubric, so it must never fall through into the tool-calling branches.
36        if should_emit_review(&request) {
37            let stream = stream::iter(vec![
38                Ok(InferenceEvent::MessageDelta(MessageDelta {
39                    text: FAKE_REVIEW_OUTPUT.to_string(),
40                    phase: None,
41                })),
42                Ok(InferenceEvent::Completed(CompletionMetadata {
43                    stop_reason: Some("stop".to_string()),
44                    provider_response_id: None,
45                })),
46            ]);
47            return Ok(Box::pin(stream));
48        }
49        if should_request_user_input(&request) {
50            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
51                ToolCallCompleted {
52                    id: "fake-user-input".to_string(),
53                    name: "request_user_input".to_string(),
54                    arguments: serde_json::json!({
55                        "questions": [{
56                            "header": "Choice",
57                            "id": "choice",
58                            "question": "Which option should be used?",
59                            "options": [
60                                { "label": "A", "description": "Use option A." },
61                                { "label": "B", "description": "Use option B." }
62                            ]
63                        }]
64                    })
65                    .to_string(),
66                },
67            ))]);
68            return Ok(tool_stream::completed(stream));
69        }
70        if should_call_external_tool(&request) {
71            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
72                ToolCallCompleted {
73                    id: "fake-external-tool".to_string(),
74                    name: "acme_lookup".to_string(),
75                    arguments: serde_json::json!({ "query": "thread status" }).to_string(),
76                },
77            ))]);
78            return Ok(tool_stream::completed(stream));
79        }
80        if should_update_task_ledger(&request) {
81            let complete = prompt_contains(&request, "FAKE_TASK_LEDGER_COMPLETE");
82            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
83                ToolCallCompleted {
84                    id: "fake-task-ledger".to_string(),
85                    name: "task_ledger.update".to_string(),
86                    arguments: task_ledger_arguments(complete),
87                },
88            ))]);
89            return Ok(tool_stream::completed(stream));
90        }
91        if let Some(tool_call) = tbench_diagnostics::next_tool_call(&request) {
92            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(tool_call))]);
93            return Ok(tool_stream::completed(stream));
94        }
95        if should_write_file(&request) {
96            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
97                ToolCallCompleted {
98                    id: "fake-write-file".to_string(),
99                    name: "write_file".to_string(),
100                    arguments: serde_json::json!({
101                        "path": "src/lib.rs",
102                        "content": "pub fn fake() -> &'static str { \"verified\" }\n"
103                    })
104                    .to_string(),
105                },
106            ))]);
107            return Ok(tool_stream::completed(stream));
108        }
109        if should_grep(&request) {
110            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
111                ToolCallCompleted {
112                    id: "fake-grep".to_string(),
113                    name: "grep".to_string(),
114                    arguments: serde_json::json!({
115                        "query": "BUG_ROOT_CAUSE_TOKEN",
116                        "path": ".",
117                        "mode": "indexed",
118                        "limit": 20
119                    })
120                    .to_string(),
121                },
122            ))]);
123            return Ok(tool_stream::completed(stream));
124        }
125        if should_zerolang_graph_dump(&request) {
126            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
127                ToolCallCompleted {
128                    id: "fake-zerolang-graph-dump".to_string(),
129                    name: "zerolang_graph_dump".to_string(),
130                    arguments: serde_json::json!({
131                        "input": "src/main.0"
132                    })
133                    .to_string(),
134                },
135            ))]);
136            return Ok(tool_stream::completed(stream));
137        }
138        if should_zerolang_edit(&request) {
139            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
140                ToolCallCompleted {
141                    id: "fake-zerolang-edit".to_string(),
142                    name: "zerolang_edit".to_string(),
143                    arguments: serde_json::json!({
144                        "input": "src/main.0",
145                        "graphHash": "graph:f76987e99677f1b3",
146                        "operations": [{
147                            "op": "set",
148                            "node": "#610c78bf",
149                            "field": "value",
150                            "expect": "hello from zero\n",
151                            "value": "hello from roder\n"
152                        }],
153                        "validate": true
154                    })
155                    .to_string(),
156                },
157            ))]);
158            return Ok(tool_stream::completed(stream));
159        }
160        if should_discovery_read(&request) {
161            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
162                ToolCallCompleted {
163                    id: "fake-discovery-read".to_string(),
164                    name: "discovery.read".to_string(),
165                    arguments: serde_json::json!({
166                        "item_id": "tool:builtin-coding-tools/grep",
167                        "promote": true,
168                        "limit": 20
169                    })
170                    .to_string(),
171                },
172            ))]);
173            return Ok(tool_stream::completed(stream));
174        }
175        if should_discovery_search(&request) {
176            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
177                ToolCallCompleted {
178                    id: "fake-discovery-search".to_string(),
179                    name: "discovery.search".to_string(),
180                    arguments: serde_json::json!({
181                        "query": "grep",
182                        "limit": 20
183                    })
184                    .to_string(),
185                },
186            ))]);
187            return Ok(tool_stream::completed(stream));
188        }
189        if should_spawn_fake_agent(&request) {
190            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
191                ToolCallCompleted {
192                    id: "fake-spawn-agent".to_string(),
193                    name: "spawn_agent".to_string(),
194                    arguments: serde_json::json!({
195                        "task_name": "reviewer",
196                        "message": "review the fake agent control smoke"
197                    })
198                    .to_string(),
199                },
200            ))]);
201            return Ok(tool_stream::completed(stream));
202        }
203        if should_list_fake_agents(&request) {
204            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
205                ToolCallCompleted {
206                    id: "fake-list-agents".to_string(),
207                    name: "list_agents".to_string(),
208                    arguments: "{}".to_string(),
209                },
210            ))]);
211            return Ok(tool_stream::completed(stream));
212        }
213        if should_message_fake_agent(&request) {
214            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
215                ToolCallCompleted {
216                    id: "fake-send-message".to_string(),
217                    name: "send_message".to_string(),
218                    arguments: serde_json::json!({
219                        "target": "reviewer",
220                        "message": "add one more fake smoke detail"
221                    })
222                    .to_string(),
223                },
224            ))]);
225            return Ok(tool_stream::completed(stream));
226        }
227        if should_wait_fake_agent(&request) {
228            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
229                ToolCallCompleted {
230                    id: "fake-wait-agent".to_string(),
231                    name: "wait_agent".to_string(),
232                    arguments: serde_json::json!({
233                        "target": "reviewer",
234                        "timeout_ms": 1000
235                    })
236                    .to_string(),
237                },
238            ))]);
239            return Ok(tool_stream::completed(stream));
240        }
241        if should_close_fake_agent(&request) {
242            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
243                ToolCallCompleted {
244                    id: "fake-close-agent".to_string(),
245                    name: "close_agent".to_string(),
246                    arguments: serde_json::json!({
247                        "target": "reviewer"
248                    })
249                    .to_string(),
250                },
251            ))]);
252            return Ok(tool_stream::completed(stream));
253        }
254        if should_complete_verification(&request) {
255            let failed = prompt_contains(&request, "FAKE_VERIFICATION_FAILED");
256            let stream = stream::iter(vec![Ok(InferenceEvent::ToolCallCompleted(
257                ToolCallCompleted {
258                    id: "fake-verification".to_string(),
259                    name: "verification_review".to_string(),
260                    arguments: verification_arguments(failed),
261                },
262            ))]);
263            return Ok(tool_stream::completed(stream));
264        }
265        if should_summarize_compaction(&request) {
266            let summary = if prompt_contains(&request, "Review the state snapshot") {
267                "<state_snapshot>goal: continue work\nprogress: fake compaction summary\nnext_steps: proceed</state_snapshot>"
268            } else {
269                "<state_snapshot>goal: continue work\nprogress: fake compaction summary\nnext_steps: proceed</state_snapshot>"
270            };
271            let stream = stream::iter(vec![
272                Ok(InferenceEvent::MessageDelta(MessageDelta {
273                    text: summary.to_string(),
274                    phase: None,
275                })),
276                Ok(InferenceEvent::Completed(CompletionMetadata {
277                    stop_reason: Some("stop".to_string()),
278                    provider_response_id: None,
279                })),
280            ]);
281            return Ok(Box::pin(stream));
282        }
283        if verification_failed(&request) {
284            let stream = stream::iter(vec![Ok(InferenceEvent::Failed(InferenceFailure {
285                message: "verification gaps remain: tests not run".to_string(),
286            }))]);
287            return Ok(Box::pin(stream));
288        }
289        if user_input_unavailable(&request) {
290            let stream = stream::iter(vec![Ok(InferenceEvent::Failed(InferenceFailure {
291                message: "clarification unavailable in non-interactive runtime profile".to_string(),
292            }))]);
293            return Ok(Box::pin(stream));
294        }
295        let stream = stream::iter(vec![
296            Ok(InferenceEvent::MessageDelta(MessageDelta {
297                text: "hello".to_string(),
298                phase: None,
299            })),
300            Ok(InferenceEvent::MessageDelta(MessageDelta {
301                text: " from".to_string(),
302                phase: None,
303            })),
304            Ok(InferenceEvent::MessageDelta(MessageDelta {
305                text: " roder".to_string(),
306                phase: None,
307            })),
308            Ok(InferenceEvent::Completed(CompletionMetadata {
309                stop_reason: Some("stop".to_string()),
310                provider_response_id: None,
311            })),
312        ]);
313
314        Ok(Box::pin(stream))
315    }
316}
317
318/// Structured review the fake provider answers with, matching the serde shape
319/// of `roder_api::review::ReviewOutput`.
320const FAKE_REVIEW_OUTPUT: &str = r#"{
321  "findings": [
322    {
323      "title": "[P1] Off-by-one in the loop bound",
324      "body": "The loop overruns the slice.",
325      "confidenceScore": 0.8,
326      "priority": "p1",
327      "codeLocation": {
328        "absoluteFilePath": "/tmp/roder-review/src/lib.rs",
329        "lineRange": { "start": 10, "end": 12 }
330      }
331    }
332  ],
333  "overallCorrectness": "patch is incorrect",
334  "overallExplanation": "One blocking bug.",
335  "overallConfidenceScore": 0.7
336}"#;
337
338fn should_emit_review(request: &AgentInferenceRequest) -> bool {
339    prompt_contains(request, "FAKE_REVIEW_FINDINGS")
340}
341
342fn should_request_user_input(request: &AgentInferenceRequest) -> bool {
343    prompt_contains(request, "FAKE_REQUEST_USER_INPUT")
344        && !request.transcript.iter().any(|item| {
345            matches!(
346                item,
347                TranscriptItem::ToolResult(result)
348                    if result.name.as_deref() == Some("request_user_input")
349            )
350        })
351}
352
353fn user_input_unavailable(request: &AgentInferenceRequest) -> bool {
354    request.transcript.iter().any(|item| {
355        matches!(
356            item,
357            TranscriptItem::ToolResult(result)
358                if result.name.as_deref() == Some("request_user_input")
359                    && result.is_error
360                    && result.result.contains("User input is unavailable")
361        )
362    })
363}
364
365fn should_call_external_tool(request: &AgentInferenceRequest) -> bool {
366    prompt_contains(request, "FAKE_EXTERNAL_TOOL") && !has_tool_result(request, "acme_lookup")
367}
368
369fn should_update_task_ledger(request: &AgentInferenceRequest) -> bool {
370    (prompt_contains(request, "FAKE_TASK_LEDGER_UPDATE")
371        || prompt_contains(request, "FAKE_TASK_LEDGER_COMPLETE"))
372        && !request.transcript.iter().any(|item| {
373            matches!(
374                item,
375                TranscriptItem::ToolResult(result)
376                    if result.name.as_deref() == Some("task_ledger.update")
377            )
378        })
379}
380
381fn should_write_file(request: &AgentInferenceRequest) -> bool {
382    prompt_contains(request, "FAKE_WRITE_FILE")
383        && !request.transcript.iter().any(|item| {
384            matches!(
385                item,
386                TranscriptItem::ToolResult(result)
387                    if result.name.as_deref() == Some("write_file")
388            )
389        })
390}
391
392fn should_grep(request: &AgentInferenceRequest) -> bool {
393    prompt_contains(request, "FAKE_GREP_INDEXED")
394        && !request.transcript.iter().any(|item| {
395            matches!(
396                item,
397                TranscriptItem::ToolResult(result) if result.name.as_deref() == Some("grep")
398            )
399        })
400}
401
402fn should_zerolang_graph_dump(request: &AgentInferenceRequest) -> bool {
403    prompt_contains(request, "FAKE_ZEROLANG_GRAPH_EDIT")
404        && !has_tool_result(request, "zerolang_graph_dump")
405}
406
407fn should_zerolang_edit(request: &AgentInferenceRequest) -> bool {
408    prompt_contains(request, "FAKE_ZEROLANG_GRAPH_EDIT")
409        && has_tool_result(request, "zerolang_graph_dump")
410        && !has_tool_result(request, "zerolang_edit")
411}
412
413fn should_discovery_search(request: &AgentInferenceRequest) -> bool {
414    prompt_contains(request, "FAKE_DISCOVERY_SEARCH")
415        && !request.transcript.iter().any(|item| {
416            matches!(
417                item,
418                TranscriptItem::ToolResult(result)
419                    if result.name.as_deref() == Some("discovery.search")
420            )
421        })
422}
423
424fn should_discovery_read(request: &AgentInferenceRequest) -> bool {
425    prompt_contains(request, "FAKE_DISCOVERY_PROMOTE")
426        && request.transcript.iter().any(|item| {
427            matches!(
428                item,
429                TranscriptItem::ToolResult(result)
430                    if result.name.as_deref() == Some("discovery.search")
431            )
432        })
433        && !request.transcript.iter().any(|item| {
434            matches!(
435                item,
436                TranscriptItem::ToolResult(result)
437                    if result.name.as_deref() == Some("discovery.read")
438            )
439        })
440}
441
442fn should_spawn_fake_agent(request: &AgentInferenceRequest) -> bool {
443    prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE") && !has_tool_result(request, "spawn_agent")
444}
445
446fn should_list_fake_agents(request: &AgentInferenceRequest) -> bool {
447    prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
448        && has_tool_result(request, "spawn_agent")
449        && !has_tool_result(request, "list_agents")
450}
451
452fn should_message_fake_agent(request: &AgentInferenceRequest) -> bool {
453    prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
454        && has_tool_result(request, "list_agents")
455        && !has_tool_result(request, "send_message")
456}
457
458fn should_wait_fake_agent(request: &AgentInferenceRequest) -> bool {
459    prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
460        && has_tool_result(request, "send_message")
461        && !has_tool_result(request, "wait_agent")
462}
463
464fn should_close_fake_agent(request: &AgentInferenceRequest) -> bool {
465    prompt_contains(request, "FAKE_AGENT_CONTROL_SMOKE")
466        && has_tool_result(request, "wait_agent")
467        && !has_tool_result(request, "close_agent")
468}
469
470fn should_summarize_compaction(request: &AgentInferenceRequest) -> bool {
471    request.metadata.get("roderCompactionSummary") == Some(&serde_json::json!(true))
472        || prompt_contains(request, crate::compaction::COMPACTION_SUMMARY_PROMPT_MARKER)
473}
474
475fn should_complete_verification(request: &AgentInferenceRequest) -> bool {
476    request.transcript.iter().any(|item| {
477        matches!(
478            item,
479            TranscriptItem::UserMessage(message)
480                if message.text.contains("Verification gate blocked final completion")
481        )
482    }) && !request.transcript.iter().any(|item| {
483        matches!(
484            item,
485            TranscriptItem::ToolResult(result)
486                if result.name.as_deref() == Some("verification_review")
487        )
488    })
489}
490
491fn verification_failed(request: &AgentInferenceRequest) -> bool {
492    request.transcript.iter().any(|item| {
493        matches!(
494            item,
495            TranscriptItem::ToolResult(result)
496                if result.name.as_deref() == Some("verification_review")
497                    && result.result.contains("Verification failed")
498        )
499    })
500}
501
502fn has_tool_result(request: &AgentInferenceRequest, name: &str) -> bool {
503    request.transcript.iter().any(|item| {
504        matches!(
505            item,
506            TranscriptItem::ToolResult(result) if result.name.as_deref() == Some(name)
507        )
508    })
509}
510
511fn prompt_contains(request: &AgentInferenceRequest, needle: &str) -> bool {
512    request.transcript.iter().any(|item| {
513        matches!(
514            item,
515            TranscriptItem::UserMessage(message) if message.text.contains(needle)
516        )
517    })
518}
519
520fn task_ledger_arguments(complete: bool) -> String {
521    let second_status = if complete { "completed" } else { "in_progress" };
522    let mut second = serde_json::json!({
523        "id": "verify",
524        "content": "Verify task",
525        "status": second_status
526    });
527    if complete {
528        second["evidence"] = serde_json::json!("fake-provider");
529    }
530    serde_json::json!({
531        "tasks": [
532            { "id": "inspect", "content": "Inspect task", "status": "completed", "evidence": "fake-provider" },
533            second
534        ],
535        "requireCompletionEvidence": true
536    })
537    .to_string()
538}
539
540fn verification_arguments(failed: bool) -> String {
541    let (status, open_gaps) = if failed {
542        ("failed", serde_json::json!(["tests not run"]))
543    } else {
544        ("completed", serde_json::json!([]))
545    };
546    serde_json::json!({
547        "originalTask": "fake verification eval",
548        "changedFiles": ["src/lib.rs"],
549        "toolEvidence": ["write_file wrote src/lib.rs"],
550        "testsRun": if failed { serde_json::json!([]) } else { serde_json::json!(["cargo test -p roder-evals verification"]) },
551        "openGaps": open_gaps,
552        "status": status
553    })
554    .to_string()
555}
roder_core/fake_provider.rs

roder_core/
fake_provider.rs
