dci-tool 0.1.0

Direct Corpus Interaction: a sandboxed, ripgrep-backed corpus-search toolset and agent for cyber-focused LLM agents, built on rig.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

//! Agentic evaluation.
//!
//! This module implements the `rig_retrieval_evals::agents::AgentEvalRunner`
//! trait for `DciAgent`. This enables end-to-end evaluation of the DCI
//! agent's full multi-turn interaction loop, tracking the final answers,
//! tool calls made, and number of turns taken to reach the answer.

use std::future::Future;
use std::pin::Pin;

use rig_core::completion::{CompletionModel, Prompt};
use rig_retrieval_evals::agents::{
    AgentEvalRunner, AgentEvalTask, AgentObservation, AgentToolCall,
};

use crate::agent::DciAgent;

impl<M: CompletionModel + 'static, P: rig_core::agent::PromptHook<M> + 'static> AgentEvalRunner
    for DciAgent<M, P>
{
    fn run<'a>(
        &'a self,
        task: &'a AgentEvalTask,
    ) -> Pin<
        Box<
            dyn Future<Output = std::result::Result<AgentObservation, rig_retrieval_evals::Error>>
                + Send
                + 'a,
        >,
    > {
        Box::pin(async move {
            let max_turns = task.max_turns.unwrap_or_else(|| self.max_turns());

            let response = self
                .agent()
                .prompt(&task.prompt)
                .max_turns(max_turns)
                .extended_details()
                .await
                .map_err(|e| {
                    rig_retrieval_evals::Error::Config(format!("Agent eval failed: {}", e))
                })?;

            let mut tool_calls = Vec::new();
            if let Some(messages) = &response.messages {
                for msg in messages {
                    if let rig_core::message::Message::Assistant { content, .. } = msg {
                        for item in content.clone() {
                            if let rig_core::completion::AssistantContent::ToolCall(tc) = item {
                                tool_calls.push(AgentToolCall {
                                    name: tc.function.name.clone(),
                                    arguments: tc.function.arguments.clone(),
                                    ok: None, // Rig doesn't explicitly track success in the Assistant message itself.
                                });
                            }
                        }
                    }
                }
            }

            Ok(AgentObservation {
                final_output: response.output,
                tool_calls,
                turns: Some(response.completion_calls.len()),
                metadata: std::collections::BTreeMap::new(),
            })
        })
    }
}