artifactize_core/
diagnostics.rs

1//! Backend readiness and tool checks.
2
3mod doctor;
4pub use doctor::{CheckStatus, DoctorReport, doctor};
5
6use std::path::Path;
7
8use serde::Serialize;
9use serde_json::json;
10use tokio_util::sync::CancellationToken;
11
12use crate::{
13    config::{Profile, RepoConfig},
14    tools::{self, human},
15    workspace,
16};
17
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, clap::ValueEnum)]
19#[serde(rename_all = "lowercase")]
20pub enum Audience {
21    Agent,
22    Human,
23}
24
25#[derive(Debug, Default)]
26pub struct ToolCheckOptions {
27    /// Select one Agent or Human eval's admitted scope (same as --eval).
28    pub selector: Option<crate::types::EvalId>,
29    /// Select one Agent or Human eval's admitted scope.
30    pub eval: Option<crate::types::EvalId>,
31    /// Check tools declared for one Artifact.
32    pub artifact: Option<crate::types::ArtifactName>,
33    /// Restrict --artifact to Agent or Human tools.
34    pub audience: Option<Audience>,
35    /// Check one tool by short or published name.
36    pub tool: Option<crate::config::ToolName>,
37    /// Invoke exactly one explicitly selected tool; never create review evidence.
38    pub execute: bool,
39    /// JSON arguments for an executed Agent tool.
40    pub args: Option<serde_json::Value>,
41}
42
43impl ToolCheckOptions {
44    pub fn validate(&self) -> Result<(), String> {
45        let eval = self.eval.is_some() || self.selector.is_some();
46        if eval && (self.artifact.is_some() || self.audience.is_some()) {
47            return Err(
48                "An eval selector determines scope and audience; do not combine it with --artifact or --audience."
49                    .into(),
50            );
51        }
52        if self.execute
53            && (self.tool.is_none()
54                || (!eval && (self.artifact.is_none() || self.audience.is_none())))
55        {
56            return Err(
57                "--execute requires --tool, with an eval selector or with --artifact and --audience."
58                    .into(),
59            );
60        }
61        if self.args.is_some() && (!self.execute || self.audience == Some(Audience::Human)) {
62            return Err(
63                "--args requires Agent tool execution; Human tools take no free arguments.".into(),
64            );
65        }
66        Ok(())
67    }
68}
69
70#[derive(Debug, Serialize)]
71#[serde(rename_all = "camelCase")]
72pub struct ToolCheckReport {
73    pub ok: bool,
74    pub scopes: Vec<ToolCheckScope>,
75    pub checks: Vec<ToolCheck>,
76    #[serde(skip_serializing_if = "Option::is_none")]
77    pub result: Option<ToolCheckResult>,
78}
79
80#[derive(Debug, Serialize)]
81#[serde(rename_all = "camelCase")]
82pub struct ToolCheckScope {
83    pub eval_id: Option<crate::config::EvalId>,
84    pub artifact_id: Option<crate::config::ArtifactName>,
85    pub audience: Audience,
86    pub tools: Vec<tools::ToolDefinition>,
87}
88#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
89#[serde(rename_all = "lowercase")]
90pub enum ToolCheckStage {
91    Declarations,
92    Preflight,
93    Execute,
94}
95
96#[derive(Debug, Serialize)]
97pub struct ToolCheck {
98    pub stage: ToolCheckStage,
99    pub tool: Option<crate::config::ToolName>,
100    pub ok: bool,
101    pub message: Option<String>,
102}
103#[derive(Debug, Serialize)]
104#[serde(untagged)]
105pub enum ToolCheckResult {
106    Agent(tools::ToolResult),
107    Human(human::ToolResult),
108}
109
110impl ToolCheckReport {
111    fn check(
112        &mut self,
113        stage: ToolCheckStage,
114        name: Option<&crate::config::ToolName>,
115        result: Result<(), String>,
116    ) {
117        self.ok &= result.is_ok();
118        self.checks.push(ToolCheck {
119            stage,
120            tool: name.cloned(),
121            ok: result.is_ok(),
122            message: result.err(),
123        });
124    }
125}
126
127enum Catalog<'a> {
128    Agent(tools::Registry<'a>),
129    Human(human::Registry<'a>),
130}
131
132impl Catalog<'_> {
133    fn definitions(&self) -> Vec<tools::ToolDefinition> {
134        match self {
135            Self::Agent(registry) => registry.list().cloned().collect(),
136            Self::Human(registry) => registry
137                .list()
138                .map(|tool| tools::ToolDefinition {
139                    name: tool.name.clone(),
140                    artifact_id: tool.artifact_id.clone(),
141                    description: tool.description.clone(),
142                    input_schema: json!({
143                        "type":"object",
144                        "properties":{},
145                        "additionalProperties":false,
146                    }),
147                })
148                .collect(),
149        }
150    }
151    fn preflight(&self, name: &crate::config::ToolName) -> Result<(), String> {
152        match self {
153            Self::Agent(r) => r.preflight(name),
154            Self::Human(r) => r.preflight(name),
155        }
156    }
157}
158
159pub async fn check_tools(
160    repo: &Path,
161    state: Option<&Path>,
162    options: &ToolCheckOptions,
163    cancellation: CancellationToken,
164) -> Result<ToolCheckReport, String> {
165    options.validate()?;
166    let mut report = ToolCheckReport {
167        ok: true,
168        scopes: Vec::new(),
169        checks: Vec::new(),
170        result: None,
171    };
172    let config = match crate::config::read_workspace_config(repo) {
173        Ok(config) => config,
174        Err(error) => {
175            report.check(ToolCheckStage::Declarations, None, Err(error.to_string()));
176            return Ok(report);
177        }
178    };
179    let selected = options.eval.as_ref().or(options.selector.as_ref());
180    if let Some(id) = selected
181        && !config.evals.iter().any(|eval| {
182            &eval.id == id
183                && matches!(
184                    eval.declaration.profile(),
185                    Profile::Agent { .. } | Profile::Human {}
186                )
187        })
188    {
189        return Err("Select a declared Agent or Human eval.".into());
190    }
191    if options.args.is_some()
192        && let Some(id) = selected
193        && config
194            .evals
195            .iter()
196            .any(|eval| &eval.id == id && matches!(eval.declaration.profile(), Profile::Human {}))
197    {
198        return Err(
199            "--args requires Agent tool execution; Human tools take no free arguments.".into(),
200        );
201    }
202    if let Some(id) = &options.artifact
203        && !config.artifacts.contains_key(id)
204    {
205        return Err(format!("Unknown Artifact: {id}"));
206    }
207    let mut catalogs = Vec::new();
208    // An eval selector keeps the eval's own scope, also for one selected tool.
209    if options.artifact.is_some()
210        || options.audience.is_some()
211        || (options.tool.is_some() && selected.is_none())
212    {
213        let artifacts: Vec<_> = options.artifact.as_ref().map_or_else(
214            || config.artifacts.keys().map(|name| name.as_str()).collect(),
215            |id| vec![id.as_str()],
216        );
217        for artifact in artifacts {
218            for audience in [Audience::Agent, Audience::Human] {
219                if options.audience.is_some_and(|a| a != audience) {
220                    continue;
221                }
222                let catalog = match audience {
223                    Audience::Agent => {
224                        tools::Registry::for_artifact(&config, artifact).map(Catalog::Agent)
225                    }
226                    Audience::Human => {
227                        human::Registry::for_artifact(&config, artifact).map(Catalog::Human)
228                    }
229                };
230                catalogs.push((None, Some(artifact), audience, catalog));
231            }
232        }
233    } else {
234        for eval in &config.evals {
235            if selected.is_some_and(|id| id != &eval.id) {
236                continue;
237            }
238            let (audience, catalog) = match eval.declaration.profile() {
239                Profile::Agent { .. } => (
240                    Audience::Agent,
241                    tools::Registry::new(&config, &eval.id).map(Catalog::Agent),
242                ),
243                Profile::Human { .. } => (
244                    Audience::Human,
245                    human::Registry::new(&config, &eval.id).map(Catalog::Human),
246                ),
247                Profile::Runtime { .. } => continue,
248            };
249            catalogs.push((Some(&eval.id), None, audience, catalog));
250        }
251    }
252    for (eval, artifact, audience, catalog) in catalogs {
253        let catalog = match catalog {
254            Ok(catalog) => catalog,
255            Err(error) => {
256                report.check(ToolCheckStage::Preflight, None, Err(error));
257                continue;
258            }
259        };
260        let mut definitions = catalog.definitions();
261        if let Some(name) = &options.tool {
262            let published = definitions.iter().any(|d| d.name == *name);
263            definitions.retain(|d| {
264                artifact.is_none_or(|id| d.artifact_id == id)
265                    && (d.name == *name
266                        || (!published
267                            && artifact.is_some_and(|id| d.name == format!("{name}_{id}"))))
268            });
269            if definitions.is_empty() {
270                report.check(
271                    ToolCheckStage::Preflight,
272                    Some(name),
273                    Err(if eval.is_some() {
274                        "The selected tool is not published to this eval.".into()
275                    } else {
276                        "The selected tool is not registered for this Artifact and audience.".into()
277                    }),
278                );
279            }
280        }
281        for definition in &definitions {
282            report.check(
283                ToolCheckStage::Preflight,
284                Some(&definition.name),
285                catalog.preflight(&definition.name),
286            );
287        }
288        report.scopes.push(ToolCheckScope {
289            eval_id: eval.cloned(),
290            artifact_id: artifact.map(|id| config.artifacts[id].name.clone()),
291            audience,
292            tools: definitions.clone(),
293        });
294        if options.execute && report.ok && definitions.len() == 1 {
295            let name = &definitions[0].name;
296            let result = execute_tool(
297                &config,
298                state,
299                &catalog,
300                name,
301                options,
302                cancellation.clone(),
303            )
304            .await;
305            match result {
306                Ok(result) => {
307                    report.check(
308                        ToolCheckStage::Execute,
309                        Some(name),
310                        if result.is_error() {
311                            Err("The tool returned an error; no review result was created.".into())
312                        } else {
313                            Ok(())
314                        },
315                    );
316                    report.result = Some(result);
317                }
318                Err(error) => report.check(ToolCheckStage::Execute, Some(name), Err(error)),
319            }
320        }
321    }
322    Ok(report)
323}
324
325async fn execute_tool(
326    config: &RepoConfig,
327    state: Option<&Path>,
328    catalog: &Catalog<'_>,
329    name: &crate::config::ToolName,
330    options: &ToolCheckOptions,
331    cancellation: CancellationToken,
332) -> Result<ToolCheckResult, String> {
333    match catalog {
334        Catalog::Human(registry) => Ok(ToolCheckResult::Human(
335            registry.call(name, cancellation).await,
336        )),
337        Catalog::Agent(registry) => {
338            let state = crate::store::state_dir(state)?;
339            let root =
340                workspace::prepare_directory(&state, &config.root).map_err(|e| e.to_string())?;
341            let directory = tempfile::Builder::new()
342                .prefix("tools-check-")
343                .tempdir_in(root)
344                .map_err(|e| e.to_string())?;
345            let args = options.args.clone().unwrap_or(json!({}));
346            let sessions = registry.sessions(&state, Some(name), cancellation.clone());
347            let registry = registry.with_session_scope(&sessions)?;
348            let result = registry
349                .call(name, args, directory.path(), cancellation)
350                .await;
351            sessions.close().await?;
352            directory
353                .close()
354                .map_err(|_| "Tool-check cleanup failed.".to_owned())?;
355            Ok(ToolCheckResult::Agent(result))
356        }
357    }
358}
359
360impl ToolCheckResult {
361    fn is_error(&self) -> bool {
362        match self {
363            Self::Agent(result) => result.is_error(),
364            Self::Human(result) => result.is_error(),
365        }
366    }
367}
artifactize_core/diagnostics.rs

artifactize_core/
diagnostics.rs
