artifactize_core/
diagnostics.rs

1//! Backend readiness and tool checks.
2
3mod doctor;
4pub use doctor::{CheckStatus, DoctorReport, doctor};
5
6use std::path::Path;
7
8use serde::Serialize;
9use serde_json::json;
10use tokio_util::sync::CancellationToken;
11
12use crate::{
13    config::{Profile, RepoConfig},
14    tools::{self, human},
15    workspace,
16};
17
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, clap::ValueEnum)]
19#[serde(rename_all = "lowercase")]
20pub enum Audience {
21    Agent,
22    Human,
23}
24
25#[derive(Debug, Default, clap::Args)]
26pub struct ToolCheckOptions {
27    /// Select one Agent or Human eval's admitted scope (same as --eval).
28    #[arg(conflicts_with = "eval")]
29    pub selector: Option<crate::types::EvalId>,
30    /// Select one Agent or Human eval's admitted scope.
31    #[arg(long, value_name = "ID")]
32    pub eval: Option<crate::types::EvalId>,
33    /// Check tools declared for one Artifact.
34    #[arg(long, value_name = "ID")]
35    pub artifact: Option<crate::types::ArtifactName>,
36    /// Restrict --artifact to Agent or Human tools.
37    #[arg(long, value_enum)]
38    pub audience: Option<Audience>,
39    /// Check one tool by short or published name.
40    #[arg(long, value_name = "NAME")]
41    pub tool: Option<crate::config::ToolName>,
42    /// Invoke exactly one explicitly selected tool; never create review evidence.
43    #[arg(long)]
44    pub execute: bool,
45    /// JSON arguments for an executed Agent tool.
46    #[arg(long, value_name = "JSON", value_parser = json_arguments)]
47    pub args: Option<serde_json::Value>,
48}
49
50impl ToolCheckOptions {
51    pub fn validate(&self) -> Result<(), String> {
52        if (self.eval.is_some() || self.selector.is_some())
53            && (self.artifact.is_some()
54                || self.audience.is_some()
55                || self.tool.is_some()
56                || self.execute)
57        {
58            return Err(
59                "An eval selector determines scope and audience; do not combine it with --artifact, --audience, --tool or --execute."
60                    .into(),
61            );
62        }
63        if self.execute
64            && (self.artifact.is_none() || self.audience.is_none() || self.tool.is_none())
65        {
66            return Err("--execute requires --artifact, --audience and --tool.".into());
67        }
68        if self.args.is_some() && (!self.execute || self.audience == Some(Audience::Human)) {
69            return Err(
70                "--args requires Agent tool execution; Human tools take no free arguments.".into(),
71            );
72        }
73        Ok(())
74    }
75}
76
77/// `--args`, parsed once where it is given.
78fn json_arguments(text: &str) -> Result<serde_json::Value, String> {
79    serde_json::from_str(text).map_err(|e| format!("Invalid --args JSON: {e}"))
80}
81
82#[derive(Debug, Serialize)]
83#[serde(rename_all = "camelCase")]
84pub struct ToolCheckReport {
85    pub ok: bool,
86    pub scopes: Vec<ToolCheckScope>,
87    pub checks: Vec<ToolCheck>,
88    #[serde(skip_serializing_if = "Option::is_none")]
89    pub result: Option<ToolCheckResult>,
90}
91
92#[derive(Debug, Serialize)]
93#[serde(rename_all = "camelCase")]
94pub struct ToolCheckScope {
95    pub eval_id: Option<crate::config::EvalId>,
96    pub artifact_id: Option<crate::config::ArtifactName>,
97    pub audience: Audience,
98    pub tools: Vec<tools::ToolDefinition>,
99}
100#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
101#[serde(rename_all = "lowercase")]
102pub enum ToolCheckStage {
103    Declarations,
104    Preflight,
105    Execute,
106}
107
108#[derive(Debug, Serialize)]
109pub struct ToolCheck {
110    pub stage: ToolCheckStage,
111    pub tool: Option<crate::config::ToolName>,
112    pub ok: bool,
113    pub message: Option<String>,
114}
115#[derive(Debug, Serialize)]
116#[serde(untagged)]
117pub enum ToolCheckResult {
118    Agent(tools::ToolResult),
119    Human(human::ToolResult),
120}
121
122impl ToolCheckReport {
123    fn check(
124        &mut self,
125        stage: ToolCheckStage,
126        name: Option<&crate::config::ToolName>,
127        result: Result<(), String>,
128    ) {
129        self.ok &= result.is_ok();
130        self.checks.push(ToolCheck {
131            stage,
132            tool: name.cloned(),
133            ok: result.is_ok(),
134            message: result.err(),
135        });
136    }
137}
138
139enum Catalog<'a> {
140    Agent(tools::Registry<'a>),
141    Human(human::Registry<'a>),
142}
143
144impl Catalog<'_> {
145    fn definitions(&self) -> Vec<tools::ToolDefinition> {
146        match self {
147            Self::Agent(registry) => registry.list().cloned().collect(),
148            Self::Human(registry) => registry
149                .list()
150                .map(|tool| tools::ToolDefinition {
151                    name: tool.name.clone(),
152                    artifact_id: tool.artifact_id.clone(),
153                    description: tool.description.clone(),
154                    input_schema: json!({
155                        "type":"object",
156                        "properties":{},
157                        "additionalProperties":false,
158                    }),
159                })
160                .collect(),
161        }
162    }
163    fn preflight(&self, name: &crate::config::ToolName) -> Result<(), String> {
164        match self {
165            Self::Agent(r) => r.preflight(name),
166            Self::Human(r) => r.preflight(name),
167        }
168    }
169}
170
171pub async fn check_tools(
172    repo: &Path,
173    state: Option<&Path>,
174    options: &ToolCheckOptions,
175    cancellation: CancellationToken,
176) -> Result<ToolCheckReport, String> {
177    options.validate()?;
178    let mut report = ToolCheckReport {
179        ok: true,
180        scopes: Vec::new(),
181        checks: Vec::new(),
182        result: None,
183    };
184    let config = match crate::config::read_workspace_config(repo) {
185        Ok(config) => config,
186        Err(error) => {
187            report.check(ToolCheckStage::Declarations, None, Err(error.to_string()));
188            return Ok(report);
189        }
190    };
191    let selected = options.eval.as_ref().or(options.selector.as_ref());
192    if let Some(id) = selected
193        && !config.evals.iter().any(|eval| {
194            &eval.id == id
195                && matches!(
196                    eval.declaration.profile(),
197                    Profile::Agent { .. } | Profile::Human {}
198                )
199        })
200    {
201        return Err("Select a declared Agent or Human eval.".into());
202    }
203    if let Some(id) = &options.artifact
204        && !config.artifacts.contains_key(id)
205    {
206        return Err(format!("Unknown Artifact: {id}"));
207    }
208    let mut catalogs = Vec::new();
209    if options.artifact.is_some() || options.audience.is_some() || options.tool.is_some() {
210        let artifacts: Vec<_> = options.artifact.as_ref().map_or_else(
211            || config.artifacts.keys().map(|name| name.as_str()).collect(),
212            |id| vec![id.as_str()],
213        );
214        for artifact in artifacts {
215            for audience in [Audience::Agent, Audience::Human] {
216                if options.audience.is_some_and(|a| a != audience) {
217                    continue;
218                }
219                let catalog = match audience {
220                    Audience::Agent => {
221                        tools::Registry::for_artifact(&config, artifact).map(Catalog::Agent)
222                    }
223                    Audience::Human => {
224                        human::Registry::for_artifact(&config, artifact).map(Catalog::Human)
225                    }
226                };
227                catalogs.push((None, Some(artifact), audience, catalog));
228            }
229        }
230    } else {
231        for eval in &config.evals {
232            if selected.is_some_and(|id| id != &eval.id) {
233                continue;
234            }
235            let (audience, catalog) = match eval.declaration.profile() {
236                Profile::Agent { .. } => (
237                    Audience::Agent,
238                    tools::Registry::new(&config, &eval.id).map(Catalog::Agent),
239                ),
240                Profile::Human { .. } => (
241                    Audience::Human,
242                    human::Registry::new(&config, &eval.id).map(Catalog::Human),
243                ),
244                Profile::Runtime { .. } | Profile::Dependency { .. } => continue,
245            };
246            catalogs.push((Some(&eval.id), None, audience, catalog));
247        }
248    }
249    for (eval, artifact, audience, catalog) in catalogs {
250        let catalog = match catalog {
251            Ok(catalog) => catalog,
252            Err(error) => {
253                report.check(ToolCheckStage::Preflight, None, Err(error));
254                continue;
255            }
256        };
257        let mut definitions = catalog.definitions();
258        if let Some(name) = &options.tool {
259            let published = definitions.iter().any(|d| d.name == *name);
260            definitions.retain(|d| {
261                artifact.is_none_or(|id| d.artifact_id == id)
262                    && (d.name == *name
263                        || (!published
264                            && artifact.is_some_and(|id| d.name == format!("{name}_{id}"))))
265            });
266            if definitions.is_empty() {
267                report.check(
268                    ToolCheckStage::Preflight,
269                    Some(name),
270                    Err(
271                        "The selected tool is not registered for this Artifact and audience."
272                            .into(),
273                    ),
274                );
275            }
276        }
277        for definition in &definitions {
278            report.check(
279                ToolCheckStage::Preflight,
280                Some(&definition.name),
281                catalog.preflight(&definition.name),
282            );
283        }
284        report.scopes.push(ToolCheckScope {
285            eval_id: eval.cloned(),
286            artifact_id: artifact.map(|id| config.artifacts[id].name.clone()),
287            audience,
288            tools: definitions.clone(),
289        });
290        if options.execute && report.ok && definitions.len() == 1 {
291            let name = &definitions[0].name;
292            let result = execute_tool(
293                &config,
294                state,
295                &catalog,
296                name,
297                options,
298                cancellation.clone(),
299            )
300            .await;
301            match result {
302                Ok(result) => {
303                    report.check(
304                        ToolCheckStage::Execute,
305                        Some(name),
306                        if result.is_error() {
307                            Err("The tool returned an error; no review result was created.".into())
308                        } else {
309                            Ok(())
310                        },
311                    );
312                    report.result = Some(result);
313                }
314                Err(error) => report.check(ToolCheckStage::Execute, Some(name), Err(error)),
315            }
316        }
317    }
318    Ok(report)
319}
320
321async fn execute_tool(
322    config: &RepoConfig,
323    state: Option<&Path>,
324    catalog: &Catalog<'_>,
325    name: &crate::config::ToolName,
326    options: &ToolCheckOptions,
327    cancellation: CancellationToken,
328) -> Result<ToolCheckResult, String> {
329    match catalog {
330        Catalog::Human(registry) => Ok(ToolCheckResult::Human(
331            registry.call(name, cancellation).await,
332        )),
333        Catalog::Agent(registry) => {
334            let state = crate::store::state_dir(state)?;
335            let root =
336                workspace::prepare_directory(&state, &config.root).map_err(|e| e.to_string())?;
337            let directory = tempfile::Builder::new()
338                .prefix("tools-check-")
339                .tempdir_in(root)
340                .map_err(|e| e.to_string())?;
341            let args = options.args.clone().unwrap_or(json!({}));
342            let sessions = registry
343                .start_sessions(&state, Some(name), cancellation.clone())
344                .await?;
345            let registry = registry.with_session_scope(&sessions)?;
346            let result = registry
347                .call(name, args, directory.path(), cancellation)
348                .await;
349            sessions.close().await?;
350            directory
351                .close()
352                .map_err(|_| "Tool-check cleanup failed.".to_owned())?;
353            Ok(ToolCheckResult::Agent(result))
354        }
355    }
356}
357
358impl ToolCheckResult {
359    fn is_error(&self) -> bool {
360        match self {
361            Self::Agent(result) => result.is_error(),
362            Self::Human(result) => result.is_error(),
363        }
364    }
365}
artifactize_core/diagnostics.rs

artifactize_core/
diagnostics.rs
