artifactize_core/
diagnostics.rs

1//! Backend readiness and tool checks.
2
3mod doctor;
4pub use doctor::{CheckStatus, DoctorReport, doctor};
5
6use std::path::Path;
7
8use serde::Serialize;
9use serde_json::json;
10use tokio_util::sync::CancellationToken;
11
12use crate::{
13    config::{Profile, RepoConfig},
14    tools::{self, human},
15    workspace,
16};
17
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, clap::ValueEnum)]
19#[serde(rename_all = "lowercase")]
20pub enum Audience {
21    Agent,
22    Human,
23}
24
25#[derive(Debug, Default, clap::Args)]
26pub struct ToolCheckOptions {
27    /// Select one Agent or Human eval's admitted scope (same as --eval).
28    #[arg(conflicts_with = "eval")]
29    pub selector: Option<crate::types::EvalId>,
30    /// Select one Agent or Human eval's admitted scope.
31    #[arg(long, value_name = "ID")]
32    pub eval: Option<crate::types::EvalId>,
33    /// Check tools declared for one Artifact.
34    #[arg(long, value_name = "ID")]
35    pub artifact: Option<crate::types::ArtifactName>,
36    /// Restrict --artifact to Agent or Human tools.
37    #[arg(long, value_enum)]
38    pub audience: Option<Audience>,
39    /// Check one tool by short or published name.
40    #[arg(long, value_name = "NAME")]
41    pub tool: Option<crate::config::ToolName>,
42    /// Invoke exactly one explicitly selected tool; never create review evidence.
43    #[arg(long)]
44    pub execute: bool,
45    /// JSON arguments for an executed Agent tool.
46    #[arg(long, value_name = "JSON", value_parser = json_arguments)]
47    pub args: Option<serde_json::Value>,
48}
49
50impl ToolCheckOptions {
51    pub fn validate(&self) -> Result<(), String> {
52        if (self.eval.is_some() || self.selector.is_some())
53            && (self.artifact.is_some()
54                || self.audience.is_some()
55                || self.tool.is_some()
56                || self.execute)
57        {
58            return Err(
59                "An eval selector determines scope and audience; do not combine it with --artifact, --audience, --tool or --execute."
60                    .into(),
61            );
62        }
63        if self.execute
64            && (self.artifact.is_none() || self.audience.is_none() || self.tool.is_none())
65        {
66            return Err("--execute requires --artifact, --audience and --tool.".into());
67        }
68        if self.args.is_some() && (!self.execute || self.audience == Some(Audience::Human)) {
69            return Err(
70                "--args requires Agent tool execution; Human tools take no free arguments.".into(),
71            );
72        }
73        Ok(())
74    }
75}
76
77/// `--args`, parsed once where it is given.
78fn json_arguments(text: &str) -> Result<serde_json::Value, String> {
79    serde_json::from_str(text).map_err(|e| format!("Invalid --args JSON: {e}"))
80}
81
82#[derive(Debug, Serialize)]
83#[serde(rename_all = "camelCase")]
84pub struct ToolCheckReport {
85    pub ok: bool,
86    pub scopes: Vec<ToolCheckScope>,
87    pub checks: Vec<ToolCheck>,
88    #[serde(skip_serializing_if = "Option::is_none")]
89    pub result: Option<ToolCheckResult>,
90}
91
92#[derive(Debug, Serialize)]
93#[serde(rename_all = "camelCase")]
94pub struct ToolCheckScope {
95    pub eval_id: Option<crate::config::EvalId>,
96    pub artifact_id: Option<crate::config::ArtifactName>,
97    pub audience: Audience,
98    pub tools: Vec<tools::ToolDefinition>,
99}
100#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
101#[serde(rename_all = "lowercase")]
102pub enum ToolCheckStage {
103    Declarations,
104    Preflight,
105    Execute,
106}
107
108#[derive(Debug, Serialize)]
109pub struct ToolCheck {
110    pub stage: ToolCheckStage,
111    pub tool: Option<String>,
112    pub ok: bool,
113    pub message: Option<String>,
114}
115#[derive(Debug, Serialize)]
116#[serde(untagged)]
117pub enum ToolCheckResult {
118    Agent(tools::ToolResult),
119    Human(human::ToolResult),
120}
121
122impl ToolCheckReport {
123    fn check(&mut self, stage: ToolCheckStage, name: Option<&str>, result: Result<(), String>) {
124        self.ok &= result.is_ok();
125        self.checks.push(ToolCheck {
126            stage,
127            tool: name.map(str::to_owned),
128            ok: result.is_ok(),
129            message: result.err(),
130        });
131    }
132}
133
134enum Catalog<'a> {
135    Agent(tools::Registry<'a>),
136    Human(human::Registry<'a>),
137}
138
139impl Catalog<'_> {
140    fn definitions(&self) -> Vec<tools::ToolDefinition> {
141        match self {
142            Self::Agent(registry) => registry.list().cloned().collect(),
143            Self::Human(registry) => registry
144                .list()
145                .map(|tool| tools::ToolDefinition {
146                    name: tool.name.clone(),
147                    artifact_id: tool.artifact_id.clone(),
148                    description: tool.description.clone(),
149                    input_schema: json!({
150                        "type":"object",
151                        "properties":{},
152                        "additionalProperties":false,
153                    }),
154                })
155                .collect(),
156        }
157    }
158    fn preflight(&self, name: &crate::config::ToolName) -> Result<(), String> {
159        match self {
160            Self::Agent(r) => r.preflight(name),
161            Self::Human(r) => r.preflight(name),
162        }
163    }
164}
165
166pub async fn check_tools(
167    repo: &Path,
168    state: Option<&Path>,
169    options: &ToolCheckOptions,
170    cancellation: CancellationToken,
171) -> Result<ToolCheckReport, String> {
172    options.validate()?;
173    let mut report = ToolCheckReport {
174        ok: true,
175        scopes: Vec::new(),
176        checks: Vec::new(),
177        result: None,
178    };
179    let config = match crate::config::read_workspace_config(repo) {
180        Ok(config) => config,
181        Err(error) => {
182            report.check(ToolCheckStage::Declarations, None, Err(error.to_string()));
183            return Ok(report);
184        }
185    };
186    let selected = options.eval.as_ref().or(options.selector.as_ref());
187    if let Some(id) = selected
188        && !config.evals.iter().any(|eval| {
189            &eval.id == id
190                && matches!(
191                    eval.declaration.profile(),
192                    Profile::Agent { .. } | Profile::Human {}
193                )
194        })
195    {
196        return Err("Select a declared Agent or Human eval.".into());
197    }
198    if let Some(id) = &options.artifact
199        && !config.artifacts.contains_key(id)
200    {
201        return Err(format!("Unknown Artifact: {id}"));
202    }
203    let mut catalogs = Vec::new();
204    if options.artifact.is_some() || options.audience.is_some() || options.tool.is_some() {
205        let artifacts: Vec<_> = options.artifact.as_ref().map_or_else(
206            || config.artifacts.keys().map(|name| name.as_str()).collect(),
207            |id| vec![id.as_str()],
208        );
209        for artifact in artifacts {
210            for audience in [Audience::Agent, Audience::Human] {
211                if options.audience.is_some_and(|a| a != audience) {
212                    continue;
213                }
214                let catalog = match audience {
215                    Audience::Agent => {
216                        tools::Registry::for_artifact(&config, artifact).map(Catalog::Agent)
217                    }
218                    Audience::Human => {
219                        human::Registry::for_artifact(&config, artifact).map(Catalog::Human)
220                    }
221                };
222                catalogs.push((None, Some(artifact), audience, catalog));
223            }
224        }
225    } else {
226        for eval in &config.evals {
227            if selected.is_some_and(|id| id != &eval.id) {
228                continue;
229            }
230            let (audience, catalog) = match eval.declaration.profile() {
231                Profile::Agent { .. } => (
232                    Audience::Agent,
233                    tools::Registry::new(&config, &eval.id).map(Catalog::Agent),
234                ),
235                Profile::Human { .. } => (
236                    Audience::Human,
237                    human::Registry::new(&config, &eval.id).map(Catalog::Human),
238                ),
239                Profile::Runtime { .. } | Profile::Dependency { .. } => continue,
240            };
241            catalogs.push((Some(&eval.id), None, audience, catalog));
242        }
243    }
244    for (eval, artifact, audience, catalog) in catalogs {
245        let catalog = match catalog {
246            Ok(catalog) => catalog,
247            Err(error) => {
248                report.check(ToolCheckStage::Preflight, None, Err(error));
249                continue;
250            }
251        };
252        let mut definitions = catalog.definitions();
253        if let Some(name) = &options.tool {
254            let published = definitions.iter().any(|d| d.name == *name);
255            definitions.retain(|d| {
256                artifact.is_none_or(|id| d.artifact_id == id)
257                    && (d.name == *name
258                        || (!published
259                            && artifact.is_some_and(|id| d.name == format!("{name}_{id}"))))
260            });
261            if definitions.is_empty() {
262                report.check(
263                    ToolCheckStage::Preflight,
264                    Some(name),
265                    Err(
266                        "The selected tool is not registered for this Artifact and audience."
267                            .into(),
268                    ),
269                );
270            }
271        }
272        for definition in &definitions {
273            report.check(
274                ToolCheckStage::Preflight,
275                Some(&definition.name),
276                catalog.preflight(&definition.name),
277            );
278        }
279        report.scopes.push(ToolCheckScope {
280            eval_id: eval.cloned(),
281            artifact_id: artifact.map(|id| config.artifacts[id].name.clone()),
282            audience,
283            tools: definitions.clone(),
284        });
285        if options.execute && report.ok && definitions.len() == 1 {
286            let name = &definitions[0].name;
287            let result = execute_tool(
288                &config,
289                state,
290                &catalog,
291                name,
292                options,
293                cancellation.clone(),
294            )
295            .await;
296            match result {
297                Ok(result) => {
298                    report.check(
299                        ToolCheckStage::Execute,
300                        Some(name),
301                        if result.is_error() {
302                            Err("The tool returned an error; no review result was created.".into())
303                        } else {
304                            Ok(())
305                        },
306                    );
307                    report.result = Some(result);
308                }
309                Err(error) => report.check(ToolCheckStage::Execute, Some(name), Err(error)),
310            }
311        }
312    }
313    Ok(report)
314}
315
316async fn execute_tool(
317    config: &RepoConfig,
318    state: Option<&Path>,
319    catalog: &Catalog<'_>,
320    name: &crate::config::ToolName,
321    options: &ToolCheckOptions,
322    cancellation: CancellationToken,
323) -> Result<ToolCheckResult, String> {
324    match catalog {
325        Catalog::Human(registry) => Ok(ToolCheckResult::Human(
326            registry.call(name, cancellation).await,
327        )),
328        Catalog::Agent(registry) => {
329            let state = crate::store::state_dir(state)?;
330            let root =
331                workspace::prepare_directory(&state, &config.root).map_err(|e| e.to_string())?;
332            let directory = tempfile::Builder::new()
333                .prefix("tools-check-")
334                .tempdir_in(root)
335                .map_err(|e| e.to_string())?;
336            let args = options.args.clone().unwrap_or(json!({}));
337            let sessions = registry
338                .start_sessions(&state, Some(name), cancellation.clone())
339                .await?;
340            let registry = registry.with_session_scope(&sessions)?;
341            let result = registry
342                .call(name, args, directory.path(), cancellation)
343                .await;
344            sessions.close().await?;
345            directory
346                .close()
347                .map_err(|_| "Tool-check cleanup failed.".to_owned())?;
348            Ok(ToolCheckResult::Agent(result))
349        }
350    }
351}
352
353impl ToolCheckResult {
354    fn is_error(&self) -> bool {
355        match self {
356            Self::Agent(result) => result.is_error(),
357            Self::Human(result) => result.is_error(),
358        }
359    }
360}
artifactize_core/diagnostics.rs

artifactize_core/
diagnostics.rs
