1#![recursion_limit = "256"]
2
3pub mod acp;
4pub mod cli;
5pub mod commands;
6pub mod config;
7pub mod env_guard;
8pub mod format;
9pub mod package;
10mod provider_bootstrap;
11pub mod skill_loader;
12pub mod skill_provenance;
13pub mod test_runner;
14#[doc(hidden)]
15pub mod tests;
16
17use clap::{error::ErrorKind, CommandFactory, Parser as ClapParser};
18use std::path::{Path, PathBuf};
19use std::sync::Arc;
20use std::{env, fs, process, thread};
21
22use cli::{
23 Cli, Command, CompletionShell, EvalCommand, MergeCaptainCommand, MergeCaptainMockCommand,
24 ModelInfoArgs, PackageArtifactsCommand, PackageCacheCommand, PackageCommand, PersonaCommand,
25 PersonaSupervisionCommand, ProvidersCommand, RunsCommand, ServeCommand, SkillCommand,
26 SkillKeyCommand, SkillTrustCommand, SkillsCommand, ToolCommand,
27};
28use harn_lexer::Lexer;
29use harn_parser::{DiagnosticSeverity, Parser, TypeChecker};
30
31pub const CLI_RUNTIME_STACK_SIZE: usize = 16 * 1024 * 1024;
32
33#[cfg(feature = "hostlib")]
34pub(crate) fn install_default_hostlib(vm: &mut harn_vm::Vm) {
35 let _ = harn_hostlib::install_default(vm);
36}
37
38#[cfg(not(feature = "hostlib"))]
39pub(crate) fn install_default_hostlib(_vm: &mut harn_vm::Vm) {}
40
41pub fn run() {
44 let handle = thread::Builder::new()
45 .name("harn-cli".to_string())
46 .stack_size(CLI_RUNTIME_STACK_SIZE)
47 .spawn(|| {
48 let runtime = tokio::runtime::Builder::new_multi_thread()
49 .enable_all()
50 .build()
51 .unwrap_or_else(|error| {
52 eprintln!("failed to start async runtime: {error}");
53 process::exit(1);
54 });
55 runtime.block_on(async_main());
56 })
57 .unwrap_or_else(|error| {
58 eprintln!("failed to start CLI runtime thread: {error}");
59 process::exit(1);
60 });
61
62 if let Err(payload) = handle.join() {
63 std::panic::resume_unwind(payload);
64 }
65}
66
67async fn async_main() {
68 let raw_args = normalize_serve_args(env::args().collect());
69 if raw_args.len() == 2 && raw_args[1].ends_with(".harn") {
70 provider_bootstrap::maybe_seed_ollama_for_run_file(Path::new(&raw_args[1]), false, false)
71 .await;
72 commands::run::run_file(
73 &raw_args[1],
74 false,
75 std::collections::HashSet::new(),
76 Vec::new(),
77 commands::run::CliLlmMockMode::Off,
78 None,
79 commands::run::RunProfileOptions::default(),
80 )
81 .await;
82 return;
83 }
84
85 let cli = match Cli::try_parse_from(&raw_args) {
86 Ok(cli) => cli,
87 Err(error) => {
88 if matches!(
89 error.kind(),
90 ErrorKind::DisplayHelp | ErrorKind::DisplayVersion
91 ) {
92 error.exit();
93 }
94 error.exit();
95 }
96 };
97
98 match cli.command.expect("clap requires a command") {
99 Command::Version => print_version(),
100 Command::Upgrade(args) => {
101 if let Err(error) = commands::upgrade::run(args).await {
102 eprintln!("error: {error}");
103 process::exit(1);
104 }
105 }
106 Command::Skill(args) => match args.command {
107 SkillCommand::Key(key_args) => match key_args.command {
108 SkillKeyCommand::Generate(generate) => commands::skill::run_key_generate(&generate),
109 },
110 SkillCommand::Sign(sign) => commands::skill::run_sign(&sign),
111 SkillCommand::Endorse(endorse) => commands::skill::run_endorse(&endorse),
112 SkillCommand::Verify(verify) => commands::skill::run_verify(&verify),
113 SkillCommand::WhoSigned(who_signed) => {
114 commands::skill::run_who_signed(&who_signed).await
115 }
116 SkillCommand::Trust(trust_args) => match trust_args.command {
117 SkillTrustCommand::Add(add) => commands::skill::run_trust_add(&add),
118 SkillTrustCommand::List(list) => commands::skill::run_trust_list(&list),
119 },
120 SkillCommand::New(new_args) => commands::skills::run_new(&new_args),
121 },
122 Command::Run(args) => {
123 if !args.explain_cost {
124 match (args.eval.as_deref(), args.file.as_deref()) {
125 (Some(code), None) => {
126 provider_bootstrap::maybe_seed_ollama_for_inline(
127 code,
128 args.yes,
129 args.llm_mock.is_some(),
130 )
131 .await;
132 }
133 (None, Some(file)) => {
134 provider_bootstrap::maybe_seed_ollama_for_run_file(
135 Path::new(file),
136 args.yes,
137 args.llm_mock.is_some(),
138 )
139 .await;
140 }
141 _ => {}
142 }
143 }
144 let denied =
145 commands::run::build_denied_builtins(args.deny.as_deref(), args.allow.as_deref());
146 let llm_mock_mode = if let Some(path) = args.llm_mock.as_ref() {
147 commands::run::CliLlmMockMode::Replay {
148 fixture_path: PathBuf::from(path),
149 }
150 } else if let Some(path) = args.llm_mock_record.as_ref() {
151 commands::run::CliLlmMockMode::Record {
152 fixture_path: PathBuf::from(path),
153 }
154 } else {
155 commands::run::CliLlmMockMode::Off
156 };
157 let attestation = args.attest.then(|| commands::run::RunAttestationOptions {
158 receipt_out: args.receipt_out.as_ref().map(PathBuf::from),
159 agent_id: args.attest_agent.clone(),
160 });
161 let profile_options = run_profile_options(&args.profile);
162
163 match (args.eval.as_deref(), args.file.as_deref()) {
164 (Some(code), None) => {
165 let (wrapped, tmp) = commands::run::prepare_eval_temp_file(code)
166 .unwrap_or_else(|e| command_error(&e));
167 let tmp_path: PathBuf = tmp.path().to_path_buf();
168 fs::write(&tmp_path, &wrapped).unwrap_or_else(|e| {
169 command_error(&format!("failed to write temp file for -e: {e}"))
170 });
171 let tmp_str = tmp_path.to_string_lossy().into_owned();
172 if args.explain_cost {
173 commands::run::run_explain_cost_file_with_skill_dirs(&tmp_str);
174 } else {
175 commands::run::run_file_with_skill_dirs(
176 &tmp_str,
177 args.trace,
178 denied,
179 args.argv.clone(),
180 args.skill_dir.clone(),
181 llm_mock_mode.clone(),
182 attestation.clone(),
183 profile_options.clone(),
184 )
185 .await;
186 }
187 drop(tmp);
188 }
189 (None, Some(file)) => {
190 if args.explain_cost {
191 commands::run::run_explain_cost_file_with_skill_dirs(file);
192 } else {
193 commands::run::run_file_with_skill_dirs(
194 file,
195 args.trace,
196 denied,
197 args.argv.clone(),
198 args.skill_dir.clone(),
199 llm_mock_mode,
200 attestation,
201 profile_options,
202 )
203 .await
204 }
205 }
206 (Some(_), Some(_)) => command_error(
207 "`harn run` accepts either `-e <code>` or `<file.harn>`, not both",
208 ),
209 (None, None) => {
210 command_error("`harn run` requires either `-e <code>` or `<file.harn>`")
211 }
212 }
213 }
214 Command::Check(args) => {
215 if args.provider_matrix {
216 let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
217 let extensions = package::load_runtime_extensions(&cwd);
218 package::install_runtime_extensions(&extensions);
219 commands::check::provider_matrix::run(args.format, args.filter.as_deref());
220 return;
221 }
222 if args.connector_matrix {
223 commands::check::connector_matrix::run(
224 args.format,
225 args.filter.as_deref(),
226 &args.targets,
227 );
228 return;
229 }
230 let mut target_strings: Vec<String> = args.targets.clone();
231 if args.workspace {
232 let anchor = target_strings.first().map(Path::new);
233 match package::load_workspace_config(anchor) {
234 Some((workspace, manifest_dir)) if !workspace.pipelines.is_empty() => {
235 for pipeline in &workspace.pipelines {
236 let candidate = Path::new(pipeline);
237 let resolved = if candidate.is_absolute() {
238 candidate.to_path_buf()
239 } else {
240 manifest_dir.join(candidate)
241 };
242 target_strings.push(resolved.to_string_lossy().into_owned());
243 }
244 }
245 Some(_) => command_error(
246 "--workspace requires `[workspace].pipelines` in the nearest harn.toml",
247 ),
248 None => command_error(
249 "--workspace could not find a harn.toml walking up from the target(s)",
250 ),
251 }
252 }
253 if target_strings.is_empty() {
254 command_error(
255 "`harn check` requires at least one target path, or `--workspace` with `[workspace].pipelines`",
256 );
257 }
258 for target in &target_strings {
259 if let Err(error) = package::validate_runtime_manifest_extensions(Path::new(target))
260 {
261 command_error(&format!("manifest extension validation failed: {error}"));
262 }
263 }
264 let targets: Vec<&str> = target_strings.iter().map(String::as_str).collect();
265 let files = commands::check::collect_harn_targets(&targets);
266 if files.is_empty() {
267 command_error("no .harn files found under the given target(s)");
268 }
269 let module_graph = commands::check::build_module_graph(&files);
270 let cross_file_imports = commands::check::collect_cross_file_imports(&module_graph);
271 let mut should_fail = false;
272 for file in &files {
273 let mut config = package::load_check_config(Some(file));
274 if let Some(path) = args.host_capabilities.as_ref() {
275 config.host_capabilities_path = Some(path.clone());
276 }
277 if let Some(path) = args.bundle_root.as_ref() {
278 config.bundle_root = Some(path.clone());
279 }
280 if args.strict_types {
281 config.strict_types = true;
282 }
283 if let Some(sev) = args.preflight.as_deref() {
284 config.preflight_severity = Some(sev.to_string());
285 }
286 let outcome = commands::check::check_file_inner(
287 file,
288 &config,
289 &cross_file_imports,
290 &module_graph,
291 args.invariants,
292 );
293 should_fail |= outcome.should_fail(config.strict);
294 }
295 if should_fail {
296 process::exit(1);
297 }
298 }
299 Command::Config(args) => {
300 if let Err(error) = commands::config_cmd::run(args).await {
301 command_error(&error);
302 }
303 }
304 Command::Explain(args) => {
305 let code = commands::explain::run_explain(&args);
306 if code != 0 {
307 process::exit(code);
308 }
309 }
310 Command::Contracts(args) => {
311 commands::contracts::handle_contracts_command(args).await;
312 }
313 Command::Connect(args) => {
314 commands::connect::run_connect(*args).await;
315 }
316 Command::Lint(args) => {
317 let targets: Vec<&str> = args.targets.iter().map(String::as_str).collect();
318 let files = commands::check::collect_harn_targets(&targets);
319 let prompt_files = commands::check::collect_prompt_targets(&targets);
320 if files.is_empty() && prompt_files.is_empty() {
321 command_error("no .harn or .harn.prompt files found under the given target(s)");
322 }
323 let module_graph = commands::check::build_module_graph(&files);
324 let cross_file_imports = commands::check::collect_cross_file_imports(&module_graph);
325 if args.fix {
326 for file in &files {
327 let mut config = package::load_check_config(Some(file));
328 commands::check::apply_harn_lint_config(file, &mut config);
329 let require_header = args.require_file_header
330 || commands::check::harn_lint_require_file_header(file);
331 let complexity_threshold =
332 commands::check::harn_lint_complexity_threshold(file);
333 let persona_step_allowlist =
334 commands::check::harn_lint_persona_step_allowlist(file);
335 commands::check::lint_fix_file(
336 file,
337 &config,
338 &cross_file_imports,
339 &module_graph,
340 require_header,
341 complexity_threshold,
342 &persona_step_allowlist,
343 );
344 }
345 for file in &prompt_files {
346 let threshold =
347 commands::check::harn_lint_template_variant_branch_threshold(file);
348 let disabled = commands::check::harn_lint_disabled_rules(file);
349 commands::check::lint_prompt_file_inner(file, threshold, &disabled);
354 }
355 } else {
356 let mut should_fail = false;
357 for file in &files {
358 let mut config = package::load_check_config(Some(file));
359 commands::check::apply_harn_lint_config(file, &mut config);
360 let require_header = args.require_file_header
361 || commands::check::harn_lint_require_file_header(file);
362 let complexity_threshold =
363 commands::check::harn_lint_complexity_threshold(file);
364 let persona_step_allowlist =
365 commands::check::harn_lint_persona_step_allowlist(file);
366 let outcome = commands::check::lint_file_inner(
367 file,
368 &config,
369 &cross_file_imports,
370 &module_graph,
371 require_header,
372 complexity_threshold,
373 &persona_step_allowlist,
374 );
375 should_fail |= outcome.should_fail(config.strict);
376 }
377 for file in &prompt_files {
378 let threshold =
379 commands::check::harn_lint_template_variant_branch_threshold(file);
380 let disabled = commands::check::harn_lint_disabled_rules(file);
381 let config = package::load_check_config(Some(file));
382 let outcome =
383 commands::check::lint_prompt_file_inner(file, threshold, &disabled);
384 should_fail |= outcome.should_fail(config.strict);
385 }
386 if should_fail {
387 process::exit(1);
388 }
389 }
390 }
391 Command::Fmt(args) => {
392 let targets: Vec<&str> = args.targets.iter().map(String::as_str).collect();
393 let anchor = targets.first().map(Path::new).unwrap_or(Path::new("."));
396 let loaded = match config::load_for_path(anchor) {
397 Ok(c) => c,
398 Err(e) => {
399 eprintln!("warning: {e}");
400 config::HarnConfig::default()
401 }
402 };
403 let mut opts = harn_fmt::FmtOptions::default();
404 if let Some(w) = loaded.fmt.line_width {
405 opts.line_width = w;
406 }
407 if let Some(w) = loaded.fmt.separator_width {
408 opts.separator_width = w;
409 }
410 if let Some(w) = args.line_width {
411 opts.line_width = w;
412 }
413 if let Some(w) = args.separator_width {
414 opts.separator_width = w;
415 }
416 commands::check::fmt_targets(
417 &targets,
418 commands::check::FmtMode::from_check_flag(args.check),
419 &opts,
420 );
421 }
422 Command::Test(args) => {
423 if args.target.as_deref() == Some("agents-conformance") {
424 if args.selection.is_some() {
425 command_error(
426 "`harn test agents-conformance` does not accept a second positional target; use --category instead",
427 );
428 }
429 if args.evals || args.determinism || args.record || args.replay || args.watch {
430 command_error(
431 "`harn test agents-conformance` cannot be combined with --evals, --determinism, --record, --replay, or --watch",
432 );
433 }
434 let Some(target_url) = args.agents_target.clone() else {
435 command_error("`harn test agents-conformance` requires --target <url>");
436 };
437 commands::agents_conformance::run_agents_conformance(
438 commands::agents_conformance::AgentsConformanceConfig {
439 target_url,
440 api_key: args.agents_api_key.clone(),
441 categories: args.agents_category.clone(),
442 timeout_ms: args.timeout,
443 verbose: args.verbose,
444 json: args.json,
445 json_out: args.json_out.clone(),
446 workspace_id: args.agents_workspace_id.clone(),
447 session_id: args.agents_session_id.clone(),
448 },
449 )
450 .await;
451 return;
452 }
453 if args.target.as_deref() == Some("protocols") {
454 if args.evals || args.determinism || args.record || args.replay || args.watch {
455 command_error(
456 "`harn test protocols` cannot be combined with --evals, --determinism, --record, --replay, or --watch",
457 );
458 }
459 if args.junit.is_some()
460 || args.agents_target.is_some()
461 || args.agents_api_key.is_some()
462 || !args.agents_category.is_empty()
463 || args.json
464 || args.json_out.is_some()
465 || args.agents_workspace_id.is_some()
466 || args.agents_session_id.is_some()
467 || args.parallel
468 || !args.skill_dir.is_empty()
469 {
470 command_error(
471 "`harn test protocols` accepts only --filter, --verbose, --timing, and an optional fixture selection",
472 );
473 }
474 commands::protocol_conformance::run_protocol_conformance(
475 args.selection.as_deref(),
476 args.filter.as_deref(),
477 args.verbose || args.timing,
478 );
479 return;
480 }
481 if args.evals {
482 if args.determinism || args.record || args.replay || args.watch {
483 command_error("--evals cannot be combined with --determinism, --record, --replay, or --watch");
484 }
485 if args.target.as_deref() != Some("package") || args.selection.is_some() {
486 command_error("package evals are run with `harn test package --evals`");
487 }
488 run_package_evals();
489 } else if args.determinism {
490 let cli_skill_dirs: Vec<PathBuf> =
491 args.skill_dir.iter().map(PathBuf::from).collect();
492 if args.watch {
493 command_error("--determinism cannot be combined with --watch");
494 }
495 if args.record || args.replay {
496 command_error("--determinism manages its own record/replay cycle");
497 }
498 if let Some(t) = args.target.as_deref() {
499 if t == "conformance" {
500 commands::test::run_conformance_determinism_tests(
501 t,
502 args.selection.as_deref(),
503 args.filter.as_deref(),
504 args.timeout,
505 &cli_skill_dirs,
506 )
507 .await;
508 } else if args.selection.is_some() {
509 command_error(
510 "only `harn test conformance` accepts a second positional target",
511 );
512 } else {
513 commands::test::run_determinism_tests(
514 t,
515 args.filter.as_deref(),
516 args.timeout,
517 &cli_skill_dirs,
518 )
519 .await;
520 }
521 } else {
522 let test_dir = if PathBuf::from("tests").is_dir() {
523 "tests".to_string()
524 } else {
525 command_error("no path specified and no tests/ directory found");
526 };
527 if args.selection.is_some() {
528 command_error(
529 "only `harn test conformance` accepts a second positional target",
530 );
531 }
532 commands::test::run_determinism_tests(
533 &test_dir,
534 args.filter.as_deref(),
535 args.timeout,
536 &cli_skill_dirs,
537 )
538 .await;
539 }
540 } else {
541 let cli_skill_dirs: Vec<PathBuf> =
542 args.skill_dir.iter().map(PathBuf::from).collect();
543 if args.record {
544 harn_vm::llm::set_replay_mode(
545 harn_vm::llm::LlmReplayMode::Record,
546 ".harn-fixtures",
547 );
548 } else if args.replay {
549 harn_vm::llm::set_replay_mode(
550 harn_vm::llm::LlmReplayMode::Replay,
551 ".harn-fixtures",
552 );
553 }
554
555 if let Some(t) = args.target.as_deref() {
556 if t == "conformance" {
557 commands::test::run_conformance_tests(
558 t,
559 args.selection.as_deref(),
560 args.filter.as_deref(),
561 args.junit.as_deref(),
562 args.timeout,
563 commands::test::ConformanceRunOptions {
564 verbose: args.verbose,
565 timing: args.timing,
566 differential_optimizations: args.differential_optimizations,
567 cli_skill_dirs: &cli_skill_dirs,
568 },
569 )
570 .await;
571 } else if args.selection.is_some() {
572 command_error(
573 "only `harn test conformance` accepts a second positional target",
574 );
575 } else if args.watch {
576 commands::test::run_watch_tests(
577 t,
578 args.filter.as_deref(),
579 args.timeout,
580 args.parallel,
581 &cli_skill_dirs,
582 )
583 .await;
584 } else {
585 commands::test::run_user_tests(
586 t,
587 args.filter.as_deref(),
588 args.timeout,
589 args.parallel,
590 &cli_skill_dirs,
591 )
592 .await;
593 }
594 } else {
595 let test_dir = if PathBuf::from("tests").is_dir() {
596 "tests".to_string()
597 } else {
598 command_error("no path specified and no tests/ directory found");
599 };
600 if args.selection.is_some() {
601 command_error(
602 "only `harn test conformance` accepts a second positional target",
603 );
604 }
605 if args.watch {
606 commands::test::run_watch_tests(
607 &test_dir,
608 args.filter.as_deref(),
609 args.timeout,
610 args.parallel,
611 &cli_skill_dirs,
612 )
613 .await;
614 } else {
615 commands::test::run_user_tests(
616 &test_dir,
617 args.filter.as_deref(),
618 args.timeout,
619 args.parallel,
620 &cli_skill_dirs,
621 )
622 .await;
623 }
624 }
625 }
626 }
627 Command::Init(args) => commands::init::init_project(args.name.as_deref(), args.template),
628 Command::New(args) => match commands::init::resolve_new_args(&args) {
629 Ok((name, template)) => commands::init::init_project(name.as_deref(), template),
630 Err(error) => {
631 eprintln!("error: {error}");
632 process::exit(1);
633 }
634 },
635 Command::Doctor(args) => {
636 commands::doctor::run_doctor_with_options(commands::doctor::DoctorOptions {
637 network: !args.no_network,
638 json: args.json,
639 })
640 .await
641 }
642 Command::Models(args) => commands::models::run(args).await,
643 Command::Local(args) => commands::local::run(args).await,
644 Command::Providers(args) => match args.command {
645 ProvidersCommand::Refresh(refresh) => {
646 if let Err(error) = commands::providers::run_refresh(&refresh).await {
647 command_error(&error);
648 }
649 }
650 ProvidersCommand::Validate(validate) => {
651 if let Err(error) = commands::providers::run_validate(&validate) {
652 command_error(&error);
653 }
654 }
655 ProvidersCommand::Export(export) => {
656 if let Err(error) = commands::providers::run_export(&export) {
657 command_error(&error);
658 }
659 }
660 },
661 Command::Try(args) => commands::try_cmd::run(args).await,
662 Command::Quickstart(args) => {
663 if let Err(error) = commands::quickstart::run_quickstart(&args).await {
664 command_error(&error);
665 }
666 }
667 Command::Demo(args) => {
668 let code = commands::demo::run(args).await;
669 if code != 0 {
670 process::exit(code);
671 }
672 }
673 Command::Serve(args) => match args.command {
674 ServeCommand::Acp(args) => {
675 if let Err(error) = commands::serve::run_acp_server(&args).await {
676 command_error(&error);
677 }
678 }
679 ServeCommand::A2a(args) => {
680 if let Err(error) = commands::serve::run_a2a_server(&args).await {
681 command_error(&error);
682 }
683 }
684 ServeCommand::Api(args) => {
685 if let Err(error) = commands::serve::run_api_server(&args).await {
686 command_error(&error);
687 }
688 }
689 ServeCommand::Mcp(args) => {
690 if let Err(error) = commands::serve::run_mcp_server(&args).await {
691 command_error(&error);
692 }
693 }
694 },
695 Command::Connector(args) => {
696 if let Err(error) = commands::connector::handle_connector_command(args).await {
697 eprintln!("error: {error}");
698 process::exit(1);
699 }
700 }
701 Command::Mcp(args) => commands::mcp::handle_mcp_command(&args.command).await,
702 Command::Watch(args) => {
703 let denied =
704 commands::run::build_denied_builtins(args.deny.as_deref(), args.allow.as_deref());
705 commands::run::run_watch(&args.file, denied).await;
706 }
707 Command::Portal(args) => {
708 commands::portal::run_portal(
709 &args.dir,
710 args.manifest,
711 args.persona_state_dir,
712 &args.host,
713 args.port,
714 args.open,
715 args.allow_remote_launch,
716 )
717 .await
718 }
719 Command::Trigger(args) => {
720 if let Err(error) = commands::trigger::handle(args).await {
721 eprintln!("error: {error}");
722 process::exit(1);
723 }
724 }
725 Command::Flow(args) => match commands::flow::run_flow(&args) {
726 Ok(code) => {
727 if code != 0 {
728 process::exit(code);
729 }
730 }
731 Err(error) => command_error(&error),
732 },
733 Command::Workflow(args) => match commands::workflow::handle(args) {
734 Ok(code) => {
735 if code != 0 {
736 process::exit(code);
737 }
738 }
739 Err(error) => command_error(&error),
740 },
741 Command::Supervisor(args) => {
742 if let Err(error) = commands::supervisor::handle(args).await {
743 eprintln!("error: {error}");
744 process::exit(1);
745 }
746 }
747 Command::Trace(args) => {
748 if let Err(error) = commands::trace::handle(args).await {
749 eprintln!("error: {error}");
750 process::exit(1);
751 }
752 }
753 Command::Crystallize(args) => {
754 if let Err(error) = commands::crystallize::run(args) {
755 eprintln!("error: {error}");
756 process::exit(1);
757 }
758 }
759 Command::Trust(args) | Command::TrustGraph(args) => {
760 if let Err(error) = commands::trust::handle(args).await {
761 eprintln!("error: {error}");
762 process::exit(1);
763 }
764 }
765 Command::Verify(args) => {
766 if let Err(error) = verify_provenance_receipt(&args.receipt, args.json) {
767 eprintln!("error: {error}");
768 process::exit(1);
769 }
770 }
771 Command::Completions(args) => print_completions(args.shell),
772 Command::Orchestrator(args) => {
773 if let Err(error) = commands::orchestrator::handle(args).await {
774 eprintln!("error: {error}");
775 process::exit(1);
776 }
777 }
778 Command::Playground(args) => {
779 provider_bootstrap::maybe_seed_ollama_for_playground(
780 Path::new(&args.host),
781 Path::new(&args.script),
782 args.yes,
783 args.llm.is_some(),
784 args.llm_mock.is_some(),
785 )
786 .await;
787 let llm_mock_mode = if let Some(path) = args.llm_mock.as_ref() {
788 commands::run::CliLlmMockMode::Replay {
789 fixture_path: PathBuf::from(path),
790 }
791 } else if let Some(path) = args.llm_mock_record.as_ref() {
792 commands::run::CliLlmMockMode::Record {
793 fixture_path: PathBuf::from(path),
794 }
795 } else {
796 commands::run::CliLlmMockMode::Off
797 };
798 if let Err(error) = commands::playground::run_command(args, llm_mock_mode).await {
799 eprint!("{error}");
800 process::exit(1);
801 }
802 }
803 Command::Runs(args) => match args.command {
804 RunsCommand::Inspect(inspect) => {
805 inspect_run_record(&inspect.path, inspect.compare.as_deref())
806 }
807 },
808 Command::Session(args) => commands::session::run(args),
809 Command::Replay(args) => replay_run_record(&args.path),
810 Command::Eval(args) => match args.command {
811 Some(EvalCommand::Prompt(prompt_args)) => {
812 let code = commands::eval_prompt::run(prompt_args).await;
813 if code != 0 {
814 process::exit(code);
815 }
816 }
817 Some(EvalCommand::ToolCalls(tool_calls_args)) => {
818 let code = commands::eval_tool_calls::run(tool_calls_args).await;
819 if code != 0 {
820 process::exit(code);
821 }
822 }
823 None => {
824 let Some(path) = args.path else {
825 eprintln!(
826 "error: `harn eval` requires a path or a subcommand (e.g. `prompt`)."
827 );
828 eprintln!("See `harn eval --help`.");
829 process::exit(2);
830 };
831 let llm_mock_mode = if let Some(path) = args.llm_mock.as_ref() {
832 commands::run::CliLlmMockMode::Replay {
833 fixture_path: PathBuf::from(path),
834 }
835 } else if let Some(path) = args.llm_mock_record.as_ref() {
836 commands::run::CliLlmMockMode::Record {
837 fixture_path: PathBuf::from(path),
838 }
839 } else {
840 commands::run::CliLlmMockMode::Off
841 };
842 eval_run_record(
843 &path,
844 args.compare.as_deref(),
845 args.structural_experiment.as_deref(),
846 &args.argv,
847 &llm_mock_mode,
848 )
849 }
850 },
851 Command::Repl => commands::repl::run_repl().await,
852 Command::Bench(args) => commands::bench::run(args).await,
853 Command::TestBench(args) => commands::test_bench::run(args.command).await,
854 Command::Viz(args) => commands::viz::run_viz(&args.file, args.output.as_deref()),
855 Command::Install(args) => package::install_packages(
856 args.frozen || args.locked || args.offline,
857 args.refetch.as_deref(),
858 args.offline,
859 args.json,
860 ),
861 Command::Add(args) => package::add_package_with_registry(
862 &args.name_or_spec,
863 args.alias.as_deref(),
864 args.git.as_deref(),
865 args.tag.as_deref(),
866 args.rev.as_deref(),
867 args.branch.as_deref(),
868 args.path.as_deref(),
869 args.registry.as_deref(),
870 ),
871 Command::Update(args) => {
872 package::update_packages(args.alias.as_deref(), args.all, args.json)
873 }
874 Command::Remove(args) => package::remove_package(&args.alias),
875 Command::Lock => package::lock_packages(),
876 Command::Package(args) => match args.command {
877 PackageCommand::List(list) => package::list_packages(list.json),
878 PackageCommand::Doctor(doctor) => package::doctor_packages(doctor.json),
879 PackageCommand::Search(search) => package::search_package_registry(
880 search.query.as_deref(),
881 search.registry.as_deref(),
882 search.json,
883 ),
884 PackageCommand::Info(info) => {
885 package::show_package_registry_info(&info.name, info.registry.as_deref(), info.json)
886 }
887 PackageCommand::Check(check) => {
888 package::check_package(check.package.as_deref(), check.json)
889 }
890 PackageCommand::Pack(pack) => package::pack_package(
891 pack.package.as_deref(),
892 pack.output.as_deref(),
893 pack.dry_run,
894 pack.json,
895 ),
896 PackageCommand::Docs(docs) => package::generate_package_docs(
897 docs.package.as_deref(),
898 docs.output.as_deref(),
899 docs.check,
900 ),
901 PackageCommand::Cache(cache) => match cache.command {
902 PackageCacheCommand::List => package::list_package_cache(),
903 PackageCacheCommand::Clean(clean) => package::clean_package_cache(clean.all),
904 PackageCacheCommand::Verify(verify) => {
905 package::verify_package_cache(verify.materialized)
906 }
907 },
908 PackageCommand::Outdated(args) => package::outdated_packages(
909 args.refresh,
910 args.remote,
911 args.registry.as_deref(),
912 args.json,
913 ),
914 PackageCommand::Audit(args) => {
915 package::audit_packages(args.registry.as_deref(), args.skip_materialized, args.json)
916 }
917 PackageCommand::Artifacts(args) => match args.command {
918 PackageArtifactsCommand::Manifest(manifest) => {
919 package::artifacts_manifest(manifest.output.as_deref())
920 }
921 PackageArtifactsCommand::Check(check) => {
922 package::artifacts_check(&check.manifest, check.json)
923 }
924 },
925 },
926 Command::Publish(args) => package::publish_package(
927 args.package.as_deref(),
928 args.dry_run,
929 args.registry.as_deref(),
930 args.json,
931 ),
932 Command::MergeCaptain(args) => match args.command {
933 MergeCaptainCommand::Run(run) => {
934 let code = commands::merge_captain::run_driver(&run);
935 if code != 0 {
936 process::exit(code);
937 }
938 }
939 MergeCaptainCommand::Ladder(ladder) => {
940 let code = commands::merge_captain::run_ladder(&ladder);
941 if code != 0 {
942 process::exit(code);
943 }
944 }
945 MergeCaptainCommand::Iterate(iterate) => {
946 let code = commands::merge_captain::run_iterate(&iterate);
947 if code != 0 {
948 process::exit(code);
949 }
950 }
951 MergeCaptainCommand::Audit(audit) => {
952 let code = commands::merge_captain::run_audit(&audit);
953 if code != 0 {
954 process::exit(code);
955 }
956 }
957 MergeCaptainCommand::Mock(mock) => {
958 let code = match mock {
959 MergeCaptainMockCommand::Init(args) => {
960 commands::merge_captain_mock::run_init(&args)
961 }
962 MergeCaptainMockCommand::Step(args) => {
963 commands::merge_captain_mock::run_step(&args)
964 }
965 MergeCaptainMockCommand::Status(args) => {
966 commands::merge_captain_mock::run_status(&args)
967 }
968 MergeCaptainMockCommand::Serve(args) => {
969 commands::merge_captain_mock::run_serve(&args).await
970 }
971 MergeCaptainMockCommand::Cleanup(args) => {
972 commands::merge_captain_mock::run_cleanup(&args)
973 }
974 MergeCaptainMockCommand::Scenarios => {
975 commands::merge_captain_mock::run_scenarios()
976 }
977 };
978 if code != 0 {
979 process::exit(code);
980 }
981 }
982 },
983 Command::Persona(args) => match args.command {
984 PersonaCommand::New(new) => {
985 if let Err(error) = commands::persona_scaffold::run_new(&new) {
986 eprintln!("error: {error}");
987 process::exit(1);
988 }
989 }
990 PersonaCommand::Doctor(doctor) => {
991 if let Err(error) =
992 commands::persona_doctor::run_doctor(args.manifest.as_deref(), &doctor).await
993 {
994 eprintln!("error: {error}");
995 process::exit(1);
996 }
997 }
998 PersonaCommand::Check(check) => {
999 commands::persona::run_check(args.manifest.as_deref(), &check)
1000 }
1001 PersonaCommand::List(list) => {
1002 commands::persona::run_list(args.manifest.as_deref(), &list)
1003 }
1004 PersonaCommand::Inspect(inspect) => {
1005 commands::persona::run_inspect(args.manifest.as_deref(), &inspect)
1006 }
1007 PersonaCommand::Status(status) => {
1008 if let Err(error) = commands::persona::run_status(
1009 args.manifest.as_deref(),
1010 &args.state_dir,
1011 &status,
1012 )
1013 .await
1014 {
1015 eprintln!("error: {error}");
1016 process::exit(1);
1017 }
1018 }
1019 PersonaCommand::Pause(control) => {
1020 if let Err(error) = commands::persona::run_pause(
1021 args.manifest.as_deref(),
1022 &args.state_dir,
1023 &control,
1024 )
1025 .await
1026 {
1027 eprintln!("error: {error}");
1028 process::exit(1);
1029 }
1030 }
1031 PersonaCommand::Resume(control) => {
1032 if let Err(error) = commands::persona::run_resume(
1033 args.manifest.as_deref(),
1034 &args.state_dir,
1035 &control,
1036 )
1037 .await
1038 {
1039 eprintln!("error: {error}");
1040 process::exit(1);
1041 }
1042 }
1043 PersonaCommand::Disable(control) => {
1044 if let Err(error) = commands::persona::run_disable(
1045 args.manifest.as_deref(),
1046 &args.state_dir,
1047 &control,
1048 )
1049 .await
1050 {
1051 eprintln!("error: {error}");
1052 process::exit(1);
1053 }
1054 }
1055 PersonaCommand::Tick(tick) => {
1056 if let Err(error) =
1057 commands::persona::run_tick(args.manifest.as_deref(), &args.state_dir, &tick)
1058 .await
1059 {
1060 eprintln!("error: {error}");
1061 process::exit(1);
1062 }
1063 }
1064 PersonaCommand::Trigger(trigger) => {
1065 if let Err(error) = commands::persona::run_trigger(
1066 args.manifest.as_deref(),
1067 &args.state_dir,
1068 &trigger,
1069 )
1070 .await
1071 {
1072 eprintln!("error: {error}");
1073 process::exit(1);
1074 }
1075 }
1076 PersonaCommand::Spend(spend) => {
1077 if let Err(error) =
1078 commands::persona::run_spend(args.manifest.as_deref(), &args.state_dir, &spend)
1079 .await
1080 {
1081 eprintln!("error: {error}");
1082 process::exit(1);
1083 }
1084 }
1085 PersonaCommand::Supervision(supervision) => match supervision.command {
1086 PersonaSupervisionCommand::Tail(tail) => {
1087 if let Err(error) = commands::persona_supervision::run_tail(
1088 args.manifest.as_deref(),
1089 &args.state_dir,
1090 &tail,
1091 )
1092 .await
1093 {
1094 eprintln!("error: {error}");
1095 process::exit(1);
1096 }
1097 }
1098 },
1099 },
1100 Command::ModelInfo(args) => {
1101 if !print_model_info(&args).await {
1102 process::exit(1);
1103 }
1104 }
1105 Command::ProviderCatalog(args) => print_provider_catalog(args.available_only),
1106 Command::ProviderReady(args) => {
1107 run_provider_ready(
1108 &args.provider,
1109 args.model.as_deref(),
1110 args.base_url.as_deref(),
1111 args.json,
1112 )
1113 .await
1114 }
1115 Command::ProviderProbe(args) => commands::provider::run_provider_probe(args).await,
1116 Command::ProviderToolProbe(args) => commands::provider::run_provider_tool_probe(args).await,
1117 Command::Skills(args) => match args.command {
1118 SkillsCommand::List(list) => commands::skills::run_list(&list),
1119 SkillsCommand::Inspect(inspect) => commands::skills::run_inspect(&inspect),
1120 SkillsCommand::Match(matcher) => commands::skills::run_match(&matcher),
1121 SkillsCommand::Install(install) => commands::skills::run_install(&install),
1122 SkillsCommand::New(new_args) => commands::skills::run_new(&new_args),
1123 },
1124 Command::Tool(args) => match args.command {
1125 ToolCommand::New(new_args) => {
1126 if let Err(error) = commands::tool::run_new(&new_args) {
1127 eprintln!("error: {error}");
1128 process::exit(1);
1129 }
1130 }
1131 },
1132 Command::DumpHighlightKeywords(args) => {
1133 commands::dump_highlight_keywords::run(&args.output, args.check);
1134 }
1135 Command::DumpTriggerQuickref(args) => {
1136 commands::dump_trigger_quickref::run(&args.output, args.check);
1137 }
1138 Command::DumpConnectorMatrix(args) => {
1139 commands::check::connector_matrix::run_docs(&args.output, &args.sources, args.check);
1140 }
1141 Command::DumpProtocolArtifacts(args) => {
1142 commands::dump_protocol_artifacts::run(&args.output_dir, args.check);
1143 }
1144 }
1145}
1146
1147fn run_profile_options(args: &cli::ProfileArgs) -> commands::run::RunProfileOptions {
1148 commands::run::RunProfileOptions {
1149 text: args.text,
1150 json_path: args.json_path.clone(),
1151 }
1152}
1153
1154fn print_completions(shell: CompletionShell) {
1155 let mut command = Cli::command();
1156 let shell = clap_complete::Shell::from(shell);
1157 clap_complete::generate(shell, &mut command, "harn", &mut std::io::stdout());
1158}
1159
1160fn normalize_serve_args(mut raw_args: Vec<String>) -> Vec<String> {
1161 if raw_args.len() > 2
1162 && raw_args.get(1).is_some_and(|arg| arg == "serve")
1163 && !matches!(
1164 raw_args.get(2).map(String::as_str),
1165 Some("acp" | "a2a" | "api" | "mcp" | "-h" | "--help")
1166 )
1167 {
1168 raw_args.insert(2, "a2a".to_string());
1169 }
1170 raw_args
1171}
1172
1173fn print_version() {
1174 println!(
1175 r#"
1176 ╱▔▔╲
1177 ╱ ╲ harn v{}
1178 │ ◆ │ the agent harness language
1179 │ │
1180 ╰──╯╱
1181 ╱╱
1182"#,
1183 env!("CARGO_PKG_VERSION")
1184 );
1185}
1186
1187async fn print_model_info(args: &ModelInfoArgs) -> bool {
1188 let resolved = harn_vm::llm_config::resolve_model_info(&args.model);
1189 let api_key_result = harn_vm::llm::resolve_api_key(&resolved.provider);
1190 let api_key_set = api_key_result.is_ok();
1191 let api_key = api_key_result.unwrap_or_default();
1192 let context_window =
1193 harn_vm::llm::fetch_provider_max_context(&resolved.provider, &resolved.id, &api_key).await;
1194 let readiness = local_openai_readiness(&resolved.provider, &resolved.id, &api_key).await;
1195 let catalog = harn_vm::llm_config::model_catalog_entry(&resolved.id);
1196 let runtime_context_window = catalog
1197 .as_ref()
1198 .and_then(|entry| entry.runtime_context_window);
1199 let capabilities = harn_vm::llm::capabilities::lookup(&resolved.provider, &resolved.id);
1200 let mut payload = serde_json::json!({
1201 "alias": args.model,
1202 "id": resolved.id,
1203 "provider": resolved.provider,
1204 "resolved_alias": resolved.alias,
1205 "tool_format": resolved.tool_format,
1206 "tier": resolved.tier,
1207 "api_key_set": api_key_set,
1208 "context_window": context_window,
1209 "runtime_context_window": runtime_context_window,
1210 "readiness": readiness,
1211 "catalog": catalog,
1212 "capabilities": {
1213 "native_tools": capabilities.native_tools,
1214 "defer_loading": capabilities.defer_loading,
1215 "tool_search": capabilities.tool_search,
1216 "max_tools": capabilities.max_tools,
1217 "prompt_caching": capabilities.prompt_caching,
1218 "vision": capabilities.vision,
1219 "vision_supported": capabilities.vision_supported,
1220 "audio": capabilities.audio,
1221 "pdf": capabilities.pdf,
1222 "files_api_supported": capabilities.files_api_supported,
1223 "json_schema": capabilities.json_schema,
1224 "prefers_xml_scaffolding": capabilities.prefers_xml_scaffolding,
1225 "prefers_markdown_scaffolding": capabilities.prefers_markdown_scaffolding,
1226 "structured_output_mode": capabilities.structured_output_mode,
1227 "supports_assistant_prefill": capabilities.supports_assistant_prefill,
1228 "prefers_role_developer": capabilities.prefers_role_developer,
1229 "prefers_xml_tools": capabilities.prefers_xml_tools,
1230 "thinking": !capabilities.thinking_modes.is_empty(),
1231 "thinking_block_style": capabilities.thinking_block_style,
1232 "thinking_modes": capabilities.thinking_modes,
1233 "interleaved_thinking_supported": capabilities.interleaved_thinking_supported,
1234 "anthropic_beta_features": capabilities.anthropic_beta_features,
1235 "preserve_thinking": capabilities.preserve_thinking,
1236 "server_parser": capabilities.server_parser,
1237 "honors_chat_template_kwargs": capabilities.honors_chat_template_kwargs,
1238 "recommended_endpoint": capabilities.recommended_endpoint,
1239 "text_tool_wire_format_supported": capabilities.text_tool_wire_format_supported,
1240 },
1241 "qc_default_model": harn_vm::llm_config::qc_default_model(&resolved.provider),
1242 });
1243
1244 let should_verify = args.verify || args.warm;
1245 let mut ok = true;
1246 if should_verify {
1247 if resolved.provider == "ollama" {
1248 let mut readiness = harn_vm::llm::OllamaReadinessOptions::new(resolved.id.clone());
1249 readiness.warm = args.warm;
1250 readiness.observe_loaded = true;
1251 readiness.keep_alive = args
1252 .keep_alive
1253 .as_deref()
1254 .and_then(harn_vm::llm::normalize_ollama_keep_alive);
1255 let result = harn_vm::llm::ollama_readiness(readiness).await;
1256 ok = result.valid;
1257 payload["readiness"] = serde_json::to_value(&result).unwrap_or_else(|error| {
1258 serde_json::json!({
1259 "valid": false,
1260 "status": "serialization_error",
1261 "message": format!("failed to serialize readiness result: {error}"),
1262 })
1263 });
1264 } else {
1265 ok = false;
1266 payload["readiness"] = serde_json::json!({
1267 "valid": false,
1268 "status": "unsupported_provider",
1269 "message": format!(
1270 "model-info --verify is only supported for Ollama models; resolved provider is '{}'",
1271 resolved.provider
1272 ),
1273 "provider": resolved.provider,
1274 });
1275 }
1276 }
1277
1278 println!(
1279 "{}",
1280 serde_json::to_string(&payload).unwrap_or_else(|error| {
1281 command_error(&format!("failed to serialize model info: {error}"))
1282 })
1283 );
1284 ok
1285}
1286
1287async fn local_openai_readiness(
1288 provider: &str,
1289 model: &str,
1290 api_key: &str,
1291) -> Option<serde_json::Value> {
1292 let def = harn_vm::llm_config::provider_config(provider)?;
1293 if def.auth_style != "none" || !harn_vm::llm::supports_model_readiness_probe(&def) {
1294 return None;
1295 }
1296 let readiness = harn_vm::llm::probe_openai_compatible_model(provider, model, api_key).await;
1297 Some(serde_json::json!({
1298 "valid": readiness.valid,
1299 "category": readiness.category,
1300 "message": readiness.message,
1301 "provider": readiness.provider,
1302 "model": readiness.model,
1303 "url": readiness.url,
1304 "status": readiness.status,
1305 "available_models": readiness.available_models,
1306 }))
1307}
1308
1309fn print_provider_catalog(available_only: bool) {
1310 let provider_names = if available_only {
1311 harn_vm::llm_config::available_provider_names()
1312 } else {
1313 harn_vm::llm_config::provider_names()
1314 };
1315 let providers: Vec<_> = provider_names
1316 .into_iter()
1317 .filter_map(|name| {
1318 harn_vm::llm_config::provider_config(&name).map(|def| {
1319 serde_json::json!({
1320 "name": name,
1321 "display_name": def.display_name,
1322 "icon": def.icon,
1323 "base_url": harn_vm::llm_config::resolve_base_url(&def),
1324 "base_url_env": def.base_url_env,
1325 "auth_style": def.auth_style,
1326 "auth_envs": harn_vm::llm_config::auth_env_names(&def.auth_env),
1327 "auth_available": harn_vm::llm_config::provider_key_available(&name),
1328 "features": def.features,
1329 "cost_per_1k_in": def.cost_per_1k_in,
1330 "cost_per_1k_out": def.cost_per_1k_out,
1331 "latency_p50_ms": def.latency_p50_ms,
1332 })
1333 })
1334 })
1335 .collect();
1336 let models: Vec<_> = harn_vm::llm_config::model_catalog_entries()
1337 .into_iter()
1338 .map(|(id, model)| {
1339 serde_json::json!({
1340 "id": id,
1341 "name": model.name,
1342 "provider": model.provider,
1343 "context_window": model.context_window,
1344 "runtime_context_window": model.runtime_context_window,
1345 "stream_timeout": model.stream_timeout,
1346 "capabilities": model.capabilities,
1347 "pricing": model.pricing,
1348 })
1349 })
1350 .collect();
1351 let aliases: Vec<_> = harn_vm::llm_config::alias_entries()
1352 .into_iter()
1353 .map(|(name, alias)| {
1354 serde_json::json!({
1355 "name": name,
1356 "id": alias.id,
1357 "provider": alias.provider,
1358 "tool_format": alias.tool_format,
1359 "tool_calling": harn_vm::llm_config::alias_tool_calling_entry(&name),
1360 })
1361 })
1362 .collect();
1363 let payload = serde_json::json!({
1364 "providers": providers,
1365 "known_model_names": harn_vm::llm_config::known_model_names(),
1366 "available_providers": harn_vm::llm_config::available_provider_names(),
1367 "aliases": aliases,
1368 "models": models,
1369 "qc_defaults": harn_vm::llm_config::qc_defaults(),
1370 });
1371 println!(
1372 "{}",
1373 serde_json::to_string(&payload).unwrap_or_else(|error| {
1374 command_error(&format!("failed to serialize provider catalog: {error}"))
1375 })
1376 );
1377}
1378
1379async fn run_provider_ready(
1380 provider: &str,
1381 model: Option<&str>,
1382 base_url: Option<&str>,
1383 json: bool,
1384) {
1385 let readiness =
1386 harn_vm::llm::readiness::probe_provider_readiness(provider, model, base_url).await;
1387 if json {
1388 match serde_json::to_string_pretty(&readiness) {
1389 Ok(payload) => println!("{payload}"),
1390 Err(error) => command_error(&format!("failed to serialize readiness result: {error}")),
1391 }
1392 } else if readiness.ok {
1393 println!("{}", readiness.message);
1394 } else {
1395 eprintln!("{}", readiness.message);
1396 }
1397 if !readiness.ok {
1398 process::exit(1);
1399 }
1400}
1401
1402fn command_error(message: &str) -> ! {
1403 Cli::command()
1404 .error(ErrorKind::ValueValidation, message)
1405 .exit()
1406}
1407
1408fn verify_provenance_receipt(path: &str, json: bool) -> Result<(), String> {
1409 let raw =
1410 fs::read_to_string(path).map_err(|error| format!("failed to read {path}: {error}"))?;
1411 let receipt: harn_vm::ProvenanceReceipt = serde_json::from_str(&raw)
1412 .map_err(|error| format!("failed to parse provenance receipt {path}: {error}"))?;
1413 let report = harn_vm::verify_receipt(&receipt);
1414 if json {
1415 println!(
1416 "{}",
1417 serde_json::to_string_pretty(&report).map_err(|error| error.to_string())?
1418 );
1419 } else if report.verified {
1420 println!(
1421 "verified receipt={} events={} receipt_hash={} event_root_hash={}",
1422 report.receipt_id.unwrap_or_else(|| "-".to_string()),
1423 report.event_count,
1424 report.receipt_hash.unwrap_or_else(|| "-".to_string()),
1425 report.event_root_hash.unwrap_or_else(|| "-".to_string())
1426 );
1427 } else {
1428 println!(
1429 "failed receipt={} events={}",
1430 report.receipt_id.unwrap_or_else(|| "-".to_string()),
1431 report.event_count
1432 );
1433 for error in &report.errors {
1434 println!(" {error}");
1435 }
1436 return Err("provenance receipt verification failed".to_string());
1437 }
1438 Ok(())
1439}
1440
1441fn load_run_record_or_exit(path: &Path) -> harn_vm::orchestration::RunRecord {
1442 match harn_vm::orchestration::load_run_record(path) {
1443 Ok(run) => run,
1444 Err(error) => {
1445 eprintln!("Failed to load run record: {error}");
1446 process::exit(1);
1447 }
1448 }
1449}
1450
1451fn load_eval_suite_manifest_or_exit(path: &Path) -> harn_vm::orchestration::EvalSuiteManifest {
1452 harn_vm::orchestration::load_eval_suite_manifest(path).unwrap_or_else(|error| {
1453 eprintln!("Failed to load eval manifest {}: {error}", path.display());
1454 process::exit(1);
1455 })
1456}
1457
1458fn load_eval_pack_manifest_or_exit(path: &Path) -> harn_vm::orchestration::EvalPackManifest {
1459 harn_vm::orchestration::load_eval_pack_manifest(path).unwrap_or_else(|error| {
1460 eprintln!("Failed to load eval pack {}: {error}", path.display());
1461 process::exit(1);
1462 })
1463}
1464
1465fn load_persona_eval_ladder_manifest_or_exit(
1466 path: &Path,
1467) -> harn_vm::orchestration::PersonaEvalLadderManifest {
1468 harn_vm::orchestration::load_persona_eval_ladder_manifest(path).unwrap_or_else(|error| {
1469 eprintln!(
1470 "Failed to load persona eval ladder {}: {error}",
1471 path.display()
1472 );
1473 process::exit(1);
1474 })
1475}
1476
1477fn file_looks_like_eval_manifest(path: &Path) -> bool {
1478 if path.file_name().and_then(|name| name.to_str()) == Some("harn.eval.toml") {
1479 return true;
1480 }
1481 if path.extension().and_then(|ext| ext.to_str()) == Some("toml") {
1482 let Ok(content) = fs::read_to_string(path) else {
1483 return false;
1484 };
1485 return toml::from_str::<harn_vm::orchestration::EvalPackManifest>(&content)
1486 .is_ok_and(|manifest| !manifest.cases.is_empty() || !manifest.ladders.is_empty());
1487 }
1488 let Ok(content) = fs::read_to_string(path) else {
1489 return false;
1490 };
1491 let Ok(json) = serde_json::from_str::<serde_json::Value>(&content) else {
1492 return false;
1493 };
1494 json.get("_type").and_then(|value| value.as_str()) == Some("eval_suite_manifest")
1495 || json.get("cases").is_some()
1496}
1497
1498fn file_looks_like_eval_pack_manifest(path: &Path) -> bool {
1499 if path.file_name().and_then(|name| name.to_str()) == Some("harn.eval.toml") {
1500 return true;
1501 }
1502 if path.extension().and_then(|ext| ext.to_str()) == Some("toml") {
1503 return file_looks_like_eval_manifest(path);
1504 }
1505 let Ok(content) = fs::read_to_string(path) else {
1506 return false;
1507 };
1508 let Ok(json) = serde_json::from_str::<serde_json::Value>(&content) else {
1509 return false;
1510 };
1511 json.get("version").is_some()
1512 && (json.get("cases").is_some() || json.get("ladders").is_some())
1513 && json.get("_type").and_then(|value| value.as_str()) != Some("eval_suite_manifest")
1514}
1515
1516fn file_looks_like_persona_eval_ladder_manifest(path: &Path) -> bool {
1517 let Ok(content) = fs::read_to_string(path) else {
1518 return false;
1519 };
1520 if path.extension().and_then(|ext| ext.to_str()) == Some("json") {
1521 let Ok(json) = serde_json::from_str::<serde_json::Value>(&content) else {
1522 return false;
1523 };
1524 return json.get("_type").and_then(|value| value.as_str())
1525 == Some("persona_eval_ladder_manifest")
1526 || json.get("timeout_tiers").is_some()
1527 || json.get("timeout-tiers").is_some();
1528 }
1529 toml::from_str::<harn_vm::orchestration::PersonaEvalLadderManifest>(&content).is_ok_and(
1530 |manifest| {
1531 manifest
1532 .type_name
1533 .eq_ignore_ascii_case("persona_eval_ladder_manifest")
1534 || (!manifest.timeout_tiers.is_empty() && manifest.backend.path.is_some())
1535 },
1536 )
1537}
1538
1539fn collect_run_record_paths(path: &str) -> Vec<PathBuf> {
1540 let path = Path::new(path);
1541 if path.is_file() {
1542 return vec![path.to_path_buf()];
1543 }
1544 if path.is_dir() {
1545 let mut entries: Vec<PathBuf> = fs::read_dir(path)
1546 .unwrap_or_else(|error| {
1547 eprintln!("Failed to read run directory {}: {error}", path.display());
1548 process::exit(1);
1549 })
1550 .filter_map(|entry| entry.ok().map(|entry| entry.path()))
1551 .filter(|entry| entry.extension().and_then(|ext| ext.to_str()) == Some("json"))
1552 .collect();
1553 entries.sort();
1554 return entries;
1555 }
1556 eprintln!("Run path does not exist: {}", path.display());
1557 process::exit(1);
1558}
1559
1560fn print_run_diff(diff: &harn_vm::orchestration::RunDiffReport) {
1561 println!(
1562 "Diff: {} -> {} [{} -> {}]",
1563 diff.left_run_id, diff.right_run_id, diff.left_status, diff.right_status
1564 );
1565 println!("Identical: {}", diff.identical);
1566 println!("Stage diffs: {}", diff.stage_diffs.len());
1567 println!("Tool diffs: {}", diff.tool_diffs.len());
1568 println!("Observability diffs: {}", diff.observability_diffs.len());
1569 println!("Transition delta: {}", diff.transition_count_delta);
1570 println!("Artifact delta: {}", diff.artifact_count_delta);
1571 println!("Checkpoint delta: {}", diff.checkpoint_count_delta);
1572 for stage in &diff.stage_diffs {
1573 println!("- {} [{}]", stage.node_id, stage.change);
1574 for detail in &stage.details {
1575 println!(" {}", detail);
1576 }
1577 }
1578 for tool in &diff.tool_diffs {
1579 println!("- tool {} [{}]", tool.tool_name, tool.args_hash);
1580 println!(" left: {:?}", tool.left_result);
1581 println!(" right: {:?}", tool.right_result);
1582 }
1583 for item in &diff.observability_diffs {
1584 println!("- {} [{}]", item.label, item.section);
1585 for detail in &item.details {
1586 println!(" {}", detail);
1587 }
1588 }
1589}
1590
1591fn inspect_run_record(path: &str, compare: Option<&str>) {
1592 let run = load_run_record_or_exit(Path::new(path));
1593 println!("Run: {}", run.id);
1594 println!(
1595 "Workflow: {}",
1596 run.workflow_name
1597 .clone()
1598 .unwrap_or_else(|| run.workflow_id.clone())
1599 );
1600 println!("Status: {}", run.status);
1601 println!("Task: {}", run.task);
1602 println!("Stages: {}", run.stages.len());
1603 println!("Artifacts: {}", run.artifacts.len());
1604 println!("Transitions: {}", run.transitions.len());
1605 println!("Checkpoints: {}", run.checkpoints.len());
1606 println!("HITL questions: {}", run.hitl_questions.len());
1607 if let Some(observability) = &run.observability {
1608 println!("Planner rounds: {}", observability.planner_rounds.len());
1609 println!("Research facts: {}", observability.research_fact_count);
1610 println!("Workers: {}", observability.worker_lineage.len());
1611 println!(
1612 "Action graph: {} nodes / {} edges",
1613 observability.action_graph_nodes.len(),
1614 observability.action_graph_edges.len()
1615 );
1616 println!(
1617 "Transcript pointers: {}",
1618 observability.transcript_pointers.len()
1619 );
1620 println!("Daemon events: {}", observability.daemon_events.len());
1621 }
1622 if let Some(parent_worker_id) = run
1623 .metadata
1624 .get("parent_worker_id")
1625 .and_then(|value| value.as_str())
1626 {
1627 println!("Parent worker: {}", parent_worker_id);
1628 }
1629 if let Some(parent_stage_id) = run
1630 .metadata
1631 .get("parent_stage_id")
1632 .and_then(|value| value.as_str())
1633 {
1634 println!("Parent stage: {}", parent_stage_id);
1635 }
1636 if run
1637 .metadata
1638 .get("delegated")
1639 .and_then(|value| value.as_bool())
1640 .unwrap_or(false)
1641 {
1642 println!("Delegated: true");
1643 }
1644 println!(
1645 "Pending nodes: {}",
1646 if run.pending_nodes.is_empty() {
1647 "-".to_string()
1648 } else {
1649 run.pending_nodes.join(", ")
1650 }
1651 );
1652 println!(
1653 "Replay fixture: {}",
1654 if run.replay_fixture.is_some() {
1655 "embedded"
1656 } else {
1657 "derived"
1658 }
1659 );
1660 for stage in &run.stages {
1661 let worker = stage.metadata.get("worker");
1662 let worker_suffix = worker
1663 .and_then(|value| value.get("name"))
1664 .and_then(|value| value.as_str())
1665 .map(|name| format!(" worker={name}"))
1666 .unwrap_or_default();
1667 println!(
1668 "- {} [{}] status={} outcome={} branch={}{}",
1669 stage.node_id,
1670 stage.kind,
1671 stage.status,
1672 stage.outcome,
1673 stage.branch.clone().unwrap_or_else(|| "-".to_string()),
1674 worker_suffix,
1675 );
1676 if let Some(worker) = worker {
1677 if let Some(worker_id) = worker.get("id").and_then(|value| value.as_str()) {
1678 println!(" worker_id: {}", worker_id);
1679 }
1680 if let Some(child_run_id) = worker.get("child_run_id").and_then(|value| value.as_str())
1681 {
1682 println!(" child_run_id: {}", child_run_id);
1683 }
1684 if let Some(child_run_path) = worker
1685 .get("child_run_path")
1686 .and_then(|value| value.as_str())
1687 {
1688 println!(" child_run_path: {}", child_run_path);
1689 }
1690 }
1691 }
1692 if let Some(observability) = &run.observability {
1693 for round in &observability.planner_rounds {
1694 println!(
1695 "- planner {} iterations={} llm_calls={} tools={} research_facts={}",
1696 round.node_id,
1697 round.iteration_count,
1698 round.llm_call_count,
1699 round.tool_execution_count,
1700 round.research_facts.len()
1701 );
1702 }
1703 for pointer in &observability.transcript_pointers {
1704 println!(
1705 "- transcript {} [{}] available={} {}",
1706 pointer.label,
1707 pointer.kind,
1708 pointer.available,
1709 pointer
1710 .path
1711 .clone()
1712 .unwrap_or_else(|| pointer.location.clone())
1713 );
1714 }
1715 for event in &observability.daemon_events {
1716 println!(
1717 "- daemon {} [{:?}] at {}",
1718 event.name, event.kind, event.timestamp
1719 );
1720 println!(" id: {}", event.daemon_id);
1721 println!(" persist_path: {}", event.persist_path);
1722 if let Some(summary) = &event.payload_summary {
1723 println!(" payload: {}", summary);
1724 }
1725 }
1726 }
1727 if let Some(compare_path) = compare {
1728 let baseline = load_run_record_or_exit(Path::new(compare_path));
1729 print_run_diff(&harn_vm::orchestration::diff_run_records(&baseline, &run));
1730 }
1731}
1732
1733fn replay_run_record(path: &str) {
1734 let run = load_run_record_or_exit(Path::new(path));
1735 println!("Replay: {}", run.id);
1736 for stage in &run.stages {
1737 println!(
1738 "[{}] status={} outcome={} branch={}",
1739 stage.node_id,
1740 stage.status,
1741 stage.outcome,
1742 stage.branch.clone().unwrap_or_else(|| "-".to_string())
1743 );
1744 if let Some(text) = &stage.visible_text {
1745 println!(" visible: {}", text);
1746 }
1747 if let Some(verification) = &stage.verification {
1748 println!(" verification: {}", verification);
1749 }
1750 }
1751 if let Some(transcript) = &run.transcript {
1752 println!(
1753 "Transcript events persisted: {}",
1754 transcript["events"]
1755 .as_array()
1756 .map(|v| v.len())
1757 .unwrap_or(0)
1758 );
1759 }
1760 let fixture = run
1761 .replay_fixture
1762 .clone()
1763 .unwrap_or_else(|| harn_vm::orchestration::replay_fixture_from_run(&run));
1764 let report = harn_vm::orchestration::evaluate_run_against_fixture(&run, &fixture);
1765 println!(
1766 "Embedded replay fixture: {}",
1767 if report.pass { "PASS" } else { "FAIL" }
1768 );
1769 for transition in &run.transitions {
1770 println!(
1771 "transition {} -> {} ({})",
1772 transition
1773 .from_node_id
1774 .clone()
1775 .unwrap_or_else(|| "start".to_string()),
1776 transition.to_node_id,
1777 transition
1778 .branch
1779 .clone()
1780 .unwrap_or_else(|| "default".to_string())
1781 );
1782 }
1783}
1784
1785fn eval_run_record(
1786 path: &str,
1787 compare: Option<&str>,
1788 structural_experiment: Option<&str>,
1789 argv: &[String],
1790 llm_mock_mode: &commands::run::CliLlmMockMode,
1791) {
1792 if let Some(experiment) = structural_experiment {
1793 let path_buf = PathBuf::from(path);
1794 if !path_buf.is_file() || path_buf.extension().and_then(|ext| ext.to_str()) != Some("harn")
1795 {
1796 eprintln!(
1797 "--structural-experiment currently requires a .harn pipeline path, got {}",
1798 path
1799 );
1800 process::exit(1);
1801 }
1802 if compare.is_some() {
1803 eprintln!("--compare cannot be combined with --structural-experiment");
1804 process::exit(1);
1805 }
1806 if matches!(llm_mock_mode, commands::run::CliLlmMockMode::Record { .. }) {
1807 eprintln!("--llm-mock-record cannot be combined with --structural-experiment");
1808 process::exit(1);
1809 }
1810 let path_buf = fs::canonicalize(&path_buf).unwrap_or_else(|error| {
1811 command_error(&format!(
1812 "failed to canonicalize structural eval pipeline {}: {error}",
1813 path_buf.display()
1814 ))
1815 });
1816 run_structural_experiment_eval(&path_buf, experiment, argv, llm_mock_mode);
1817 return;
1818 }
1819
1820 let path_buf = PathBuf::from(path);
1821 if path_buf.is_file() && file_looks_like_persona_eval_ladder_manifest(&path_buf) {
1822 if compare.is_some() {
1823 eprintln!("--compare is not supported with persona eval ladder manifests");
1824 process::exit(1);
1825 }
1826 let manifest = load_persona_eval_ladder_manifest_or_exit(&path_buf);
1827 let report =
1828 harn_vm::orchestration::run_persona_eval_ladder(&manifest).unwrap_or_else(|error| {
1829 eprintln!(
1830 "Failed to evaluate persona eval ladder {}: {error}",
1831 path_buf.display()
1832 );
1833 process::exit(1);
1834 });
1835 print_persona_ladder_report(&report);
1836 if !report.pass {
1837 process::exit(1);
1838 }
1839 return;
1840 }
1841
1842 if path_buf.is_file() && file_looks_like_eval_pack_manifest(&path_buf) {
1843 if compare.is_some() {
1844 eprintln!("--compare is not supported with eval pack manifests");
1845 process::exit(1);
1846 }
1847 let manifest = load_eval_pack_manifest_or_exit(&path_buf);
1848 let report = harn_vm::orchestration::evaluate_eval_pack_manifest(&manifest).unwrap_or_else(
1849 |error| {
1850 eprintln!(
1851 "Failed to evaluate eval pack {}: {error}",
1852 path_buf.display()
1853 );
1854 process::exit(1);
1855 },
1856 );
1857 print_eval_pack_report(&report);
1858 if !report.pass {
1859 process::exit(1);
1860 }
1861 return;
1862 }
1863
1864 if path_buf.is_file() && file_looks_like_eval_manifest(&path_buf) {
1865 if compare.is_some() {
1866 eprintln!("--compare is not supported with eval suite manifests");
1867 process::exit(1);
1868 }
1869 let manifest = load_eval_suite_manifest_or_exit(&path_buf);
1870 let suite = harn_vm::orchestration::evaluate_run_suite_manifest(&manifest).unwrap_or_else(
1871 |error| {
1872 eprintln!(
1873 "Failed to evaluate manifest {}: {error}",
1874 path_buf.display()
1875 );
1876 process::exit(1);
1877 },
1878 );
1879 println!(
1880 "{} {} passed, {} failed, {} total",
1881 if suite.pass { "PASS" } else { "FAIL" },
1882 suite.passed,
1883 suite.failed,
1884 suite.total
1885 );
1886 for case in &suite.cases {
1887 println!(
1888 "- {} [{}] {}",
1889 case.label.clone().unwrap_or_else(|| case.run_id.clone()),
1890 case.workflow_id,
1891 if case.pass { "PASS" } else { "FAIL" }
1892 );
1893 if let Some(path) = &case.source_path {
1894 println!(" path: {}", path);
1895 }
1896 if let Some(comparison) = &case.comparison {
1897 println!(" baseline identical: {}", comparison.identical);
1898 if !comparison.identical {
1899 println!(
1900 " baseline status: {} -> {}",
1901 comparison.left_status, comparison.right_status
1902 );
1903 }
1904 }
1905 for failure in &case.failures {
1906 println!(" {}", failure);
1907 }
1908 }
1909 if !suite.pass {
1910 process::exit(1);
1911 }
1912 return;
1913 }
1914
1915 let paths = collect_run_record_paths(path);
1916 if paths.len() > 1 {
1917 let mut cases = Vec::new();
1918 for path in &paths {
1919 let run = load_run_record_or_exit(path);
1920 let fixture = run
1921 .replay_fixture
1922 .clone()
1923 .unwrap_or_else(|| harn_vm::orchestration::replay_fixture_from_run(&run));
1924 cases.push((run, fixture, Some(path.display().to_string())));
1925 }
1926 let suite = harn_vm::orchestration::evaluate_run_suite(cases);
1927 println!(
1928 "{} {} passed, {} failed, {} total",
1929 if suite.pass { "PASS" } else { "FAIL" },
1930 suite.passed,
1931 suite.failed,
1932 suite.total
1933 );
1934 for case in &suite.cases {
1935 println!(
1936 "- {} [{}] {}",
1937 case.run_id,
1938 case.workflow_id,
1939 if case.pass { "PASS" } else { "FAIL" }
1940 );
1941 if let Some(path) = &case.source_path {
1942 println!(" path: {}", path);
1943 }
1944 if let Some(comparison) = &case.comparison {
1945 println!(" baseline identical: {}", comparison.identical);
1946 }
1947 for failure in &case.failures {
1948 println!(" {}", failure);
1949 }
1950 }
1951 if !suite.pass {
1952 process::exit(1);
1953 }
1954 return;
1955 }
1956
1957 let run = load_run_record_or_exit(&paths[0]);
1958 let fixture = run
1959 .replay_fixture
1960 .clone()
1961 .unwrap_or_else(|| harn_vm::orchestration::replay_fixture_from_run(&run));
1962 let report = harn_vm::orchestration::evaluate_run_against_fixture(&run, &fixture);
1963 println!("{}", if report.pass { "PASS" } else { "FAIL" });
1964 println!("Stages: {}", report.stage_count);
1965 if let Some(compare_path) = compare {
1966 let baseline = load_run_record_or_exit(Path::new(compare_path));
1967 print_run_diff(&harn_vm::orchestration::diff_run_records(&baseline, &run));
1968 }
1969 if !report.failures.is_empty() {
1970 for failure in &report.failures {
1971 println!("- {}", failure);
1972 }
1973 }
1974 if !report.pass {
1975 process::exit(1);
1976 }
1977}
1978
1979fn print_eval_pack_report(report: &harn_vm::orchestration::EvalPackReport) {
1980 println!(
1981 "{} {} passed, {} blocking failed, {} warning, {} informational, {} total",
1982 if report.pass { "PASS" } else { "FAIL" },
1983 report.passed,
1984 report.blocking_failed,
1985 report.warning_failed,
1986 report.informational_failed,
1987 report.total
1988 );
1989 for case in &report.cases {
1990 println!(
1991 "- {} [{}] {} ({})",
1992 case.label,
1993 case.workflow_id,
1994 if case.pass { "PASS" } else { "FAIL" },
1995 case.severity
1996 );
1997 if let Some(path) = &case.source_path {
1998 println!(" path: {}", path);
1999 }
2000 if let Some(comparison) = &case.comparison {
2001 println!(" baseline identical: {}", comparison.identical);
2002 if !comparison.identical {
2003 println!(
2004 " baseline status: {} -> {}",
2005 comparison.left_status, comparison.right_status
2006 );
2007 }
2008 }
2009 for failure in &case.failures {
2010 println!(" {}", failure);
2011 }
2012 for warning in &case.warnings {
2013 println!(" warning: {}", warning);
2014 }
2015 for item in &case.informational {
2016 println!(" info: {}", item);
2017 }
2018 }
2019 for ladder in &report.ladders {
2020 println!(
2021 "- ladder {} [{}] {} ({}) first_correct={}/{}",
2022 ladder.id,
2023 ladder.persona,
2024 if ladder.pass { "PASS" } else { "FAIL" },
2025 ladder.severity,
2026 ladder.first_correct_route.as_deref().unwrap_or("<none>"),
2027 ladder.first_correct_tier.as_deref().unwrap_or("<none>")
2028 );
2029 println!(" artifacts: {}", ladder.artifact_root);
2030 for tier in &ladder.tiers {
2031 println!(
2032 " - {} [{}] {} tools={} models={} latency={}ms cost=${:.6}",
2033 tier.timeout_tier,
2034 tier.route_id,
2035 tier.outcome,
2036 tier.tool_calls,
2037 tier.model_calls,
2038 tier.latency_ms,
2039 tier.cost_usd
2040 );
2041 for reason in &tier.degradation_reasons {
2042 println!(" {}", reason);
2043 }
2044 }
2045 }
2046}
2047
2048fn print_persona_ladder_report(report: &harn_vm::orchestration::PersonaEvalLadderReport) {
2049 println!(
2050 "{} ladder {} passed, {} degraded/looped, {} total",
2051 if report.pass { "PASS" } else { "FAIL" },
2052 report.passed,
2053 report.failed,
2054 report.total
2055 );
2056 println!(
2057 "first_correct: {}/{}",
2058 report.first_correct_route.as_deref().unwrap_or("<none>"),
2059 report.first_correct_tier.as_deref().unwrap_or("<none>")
2060 );
2061 println!("artifacts: {}", report.artifact_root);
2062 for tier in &report.tiers {
2063 println!(
2064 "- {} [{}] {} tools={} models={} latency={}ms cost=${:.6}",
2065 tier.timeout_tier,
2066 tier.route_id,
2067 tier.outcome,
2068 tier.tool_calls,
2069 tier.model_calls,
2070 tier.latency_ms,
2071 tier.cost_usd
2072 );
2073 for reason in &tier.degradation_reasons {
2074 println!(" {}", reason);
2075 }
2076 }
2077}
2078
2079fn run_package_evals() {
2080 let paths = package::load_package_eval_pack_paths(None).unwrap_or_else(|error| {
2081 eprintln!("{error}");
2082 process::exit(1);
2083 });
2084 let mut all_pass = true;
2085 for path in &paths {
2086 println!("Eval pack: {}", path.display());
2087 let manifest = load_eval_pack_manifest_or_exit(path);
2088 let report = harn_vm::orchestration::evaluate_eval_pack_manifest(&manifest).unwrap_or_else(
2089 |error| {
2090 eprintln!("Failed to evaluate eval pack {}: {error}", path.display());
2091 process::exit(1);
2092 },
2093 );
2094 print_eval_pack_report(&report);
2095 all_pass &= report.pass;
2096 }
2097 if !all_pass {
2098 process::exit(1);
2099 }
2100}
2101
2102fn run_structural_experiment_eval(
2103 path: &Path,
2104 experiment: &str,
2105 argv: &[String],
2106 llm_mock_mode: &commands::run::CliLlmMockMode,
2107) {
2108 let baseline_dir = tempfile::Builder::new()
2109 .prefix("harn-eval-baseline-")
2110 .tempdir()
2111 .unwrap_or_else(|error| {
2112 command_error(&format!("failed to create baseline tempdir: {error}"))
2113 });
2114 let variant_dir = tempfile::Builder::new()
2115 .prefix("harn-eval-variant-")
2116 .tempdir()
2117 .unwrap_or_else(|error| {
2118 command_error(&format!("failed to create variant tempdir: {error}"))
2119 });
2120
2121 let baseline = spawn_eval_pipeline_run(path, baseline_dir.path(), None, argv, llm_mock_mode);
2122 if !baseline.status.success() {
2123 relay_subprocess_failure("baseline", &baseline);
2124 }
2125
2126 let variant = spawn_eval_pipeline_run(
2127 path,
2128 variant_dir.path(),
2129 Some(experiment),
2130 argv,
2131 llm_mock_mode,
2132 );
2133 if !variant.status.success() {
2134 relay_subprocess_failure("variant", &variant);
2135 }
2136
2137 let baseline_runs = collect_structural_eval_runs(baseline_dir.path());
2138 let variant_runs = collect_structural_eval_runs(variant_dir.path());
2139 if baseline_runs.is_empty() || variant_runs.is_empty() {
2140 eprintln!(
2141 "structural eval expected workflow run records under {} and {}, but one side was empty",
2142 baseline_dir.path().display(),
2143 variant_dir.path().display()
2144 );
2145 process::exit(1);
2146 }
2147 if baseline_runs.len() != variant_runs.len() {
2148 eprintln!(
2149 "structural eval produced different run counts: baseline={} variant={}",
2150 baseline_runs.len(),
2151 variant_runs.len()
2152 );
2153 process::exit(1);
2154 }
2155
2156 let mut baseline_ok = 0usize;
2157 let mut variant_ok = 0usize;
2158 let mut any_failures = false;
2159
2160 println!("Structural experiment: {}", experiment);
2161 println!("Cases: {}", baseline_runs.len());
2162 for (baseline_run, variant_run) in baseline_runs.iter().zip(variant_runs.iter()) {
2163 let baseline_fixture = baseline_run
2164 .replay_fixture
2165 .clone()
2166 .unwrap_or_else(|| harn_vm::orchestration::replay_fixture_from_run(baseline_run));
2167 let variant_fixture = variant_run
2168 .replay_fixture
2169 .clone()
2170 .unwrap_or_else(|| harn_vm::orchestration::replay_fixture_from_run(variant_run));
2171 let baseline_report =
2172 harn_vm::orchestration::evaluate_run_against_fixture(baseline_run, &baseline_fixture);
2173 let variant_report =
2174 harn_vm::orchestration::evaluate_run_against_fixture(variant_run, &variant_fixture);
2175 let diff = harn_vm::orchestration::diff_run_records(baseline_run, variant_run);
2176 if baseline_report.pass {
2177 baseline_ok += 1;
2178 }
2179 if variant_report.pass {
2180 variant_ok += 1;
2181 }
2182 any_failures |= !baseline_report.pass || !variant_report.pass;
2183 println!(
2184 "- {} [{}]",
2185 variant_run
2186 .workflow_name
2187 .clone()
2188 .unwrap_or_else(|| variant_run.workflow_id.clone()),
2189 variant_run.task
2190 );
2191 println!(
2192 " baseline: {}",
2193 if baseline_report.pass { "PASS" } else { "FAIL" }
2194 );
2195 for failure in &baseline_report.failures {
2196 println!(" {}", failure);
2197 }
2198 println!(
2199 " variant: {}",
2200 if variant_report.pass { "PASS" } else { "FAIL" }
2201 );
2202 for failure in &variant_report.failures {
2203 println!(" {}", failure);
2204 }
2205 println!(" diff identical: {}", diff.identical);
2206 println!(" stage diffs: {}", diff.stage_diffs.len());
2207 println!(" tool diffs: {}", diff.tool_diffs.len());
2208 println!(" observability diffs: {}", diff.observability_diffs.len());
2209 }
2210
2211 println!("Baseline {} / {} passed", baseline_ok, baseline_runs.len());
2212 println!("Variant {} / {} passed", variant_ok, variant_runs.len());
2213
2214 if any_failures {
2215 process::exit(1);
2216 }
2217}
2218
2219fn spawn_eval_pipeline_run(
2220 path: &Path,
2221 run_dir: &Path,
2222 structural_experiment: Option<&str>,
2223 argv: &[String],
2224 llm_mock_mode: &commands::run::CliLlmMockMode,
2225) -> std::process::Output {
2226 let exe = env::current_exe().unwrap_or_else(|error| {
2227 command_error(&format!("failed to resolve current executable: {error}"))
2228 });
2229 let mut command = std::process::Command::new(exe);
2230 command.current_dir(path.parent().unwrap_or_else(|| Path::new(".")));
2231 command.arg("run");
2232 match llm_mock_mode {
2233 commands::run::CliLlmMockMode::Off => {}
2234 commands::run::CliLlmMockMode::Replay { fixture_path } => {
2235 command
2236 .arg("--llm-mock")
2237 .arg(absolute_cli_path(fixture_path));
2238 }
2239 commands::run::CliLlmMockMode::Record { fixture_path } => {
2240 command
2241 .arg("--llm-mock-record")
2242 .arg(absolute_cli_path(fixture_path));
2243 }
2244 }
2245 command.arg(path);
2246 if !argv.is_empty() {
2247 command.arg("--");
2248 command.args(argv);
2249 }
2250 command.env(harn_vm::runtime_paths::HARN_RUN_DIR_ENV, run_dir);
2251 if let Some(experiment) = structural_experiment {
2252 command.env("HARN_STRUCTURAL_EXPERIMENT", experiment);
2253 }
2254 command.output().unwrap_or_else(|error| {
2255 command_error(&format!(
2256 "failed to spawn `harn run {}` for structural eval: {error}",
2257 path.display()
2258 ))
2259 })
2260}
2261
2262fn absolute_cli_path(path: &Path) -> PathBuf {
2263 if path.is_absolute() {
2264 return path.to_path_buf();
2265 }
2266 env::current_dir()
2267 .unwrap_or_else(|_| PathBuf::from("."))
2268 .join(path)
2269}
2270
2271fn relay_subprocess_failure(label: &str, output: &std::process::Output) -> ! {
2272 let stdout = String::from_utf8_lossy(&output.stdout);
2273 let stderr = String::from_utf8_lossy(&output.stderr);
2274 if !stdout.trim().is_empty() {
2275 eprintln!("[{label}] stdout:\n{stdout}");
2276 }
2277 if !stderr.trim().is_empty() {
2278 eprintln!("[{label}] stderr:\n{stderr}");
2279 }
2280 process::exit(output.status.code().unwrap_or(1));
2281}
2282
2283fn collect_structural_eval_runs(dir: &Path) -> Vec<harn_vm::orchestration::RunRecord> {
2284 let mut paths: Vec<PathBuf> = fs::read_dir(dir)
2285 .unwrap_or_else(|error| {
2286 command_error(&format!(
2287 "failed to read structural eval run dir {}: {error}",
2288 dir.display()
2289 ))
2290 })
2291 .filter_map(|entry| entry.ok().map(|entry| entry.path()))
2292 .filter(|entry| entry.extension().and_then(|ext| ext.to_str()) == Some("json"))
2293 .collect();
2294 paths.sort();
2295 let mut runs: Vec<_> = paths
2296 .iter()
2297 .map(|path| load_run_record_or_exit(path))
2298 .collect();
2299 runs.sort_by(|left, right| {
2300 (
2301 left.started_at.as_str(),
2302 left.workflow_id.as_str(),
2303 left.task.as_str(),
2304 )
2305 .cmp(&(
2306 right.started_at.as_str(),
2307 right.workflow_id.as_str(),
2308 right.task.as_str(),
2309 ))
2310 });
2311 runs
2312}
2313
2314pub(crate) fn parse_source_file(path: &str) -> (String, Vec<harn_parser::SNode>) {
2316 let source = match fs::read_to_string(path) {
2317 Ok(s) => s,
2318 Err(e) => {
2319 eprintln!("Error reading {path}: {e}");
2320 process::exit(1);
2321 }
2322 };
2323
2324 let mut lexer = Lexer::new(&source);
2325 let tokens = match lexer.tokenize() {
2326 Ok(t) => t,
2327 Err(e) => {
2328 let diagnostic = harn_parser::diagnostic::render_diagnostic(
2329 &source,
2330 path,
2331 &error_span_from_lex(&e),
2332 "error",
2333 &e.to_string(),
2334 Some("here"),
2335 None,
2336 );
2337 eprint!("{diagnostic}");
2338 process::exit(1);
2339 }
2340 };
2341
2342 let mut parser = Parser::new(tokens);
2343 let program = match parser.parse() {
2344 Ok(p) => p,
2345 Err(err) => {
2346 if parser.all_errors().is_empty() {
2347 let span = error_span_from_parse(&err);
2348 let diagnostic = harn_parser::diagnostic::render_diagnostic(
2349 &source,
2350 path,
2351 &span,
2352 "error",
2353 &harn_parser::diagnostic::parser_error_message(&err),
2354 Some(harn_parser::diagnostic::parser_error_label(&err)),
2355 harn_parser::diagnostic::parser_error_help(&err),
2356 );
2357 eprint!("{diagnostic}");
2358 } else {
2359 for e in parser.all_errors() {
2360 let span = error_span_from_parse(e);
2361 let diagnostic = harn_parser::diagnostic::render_diagnostic(
2362 &source,
2363 path,
2364 &span,
2365 "error",
2366 &harn_parser::diagnostic::parser_error_message(e),
2367 Some(harn_parser::diagnostic::parser_error_label(e)),
2368 harn_parser::diagnostic::parser_error_help(e),
2369 );
2370 eprint!("{diagnostic}");
2371 }
2372 }
2373 process::exit(1);
2374 }
2375 };
2376
2377 (source, program)
2378}
2379
2380fn error_span_from_lex(e: &harn_lexer::LexerError) -> harn_lexer::Span {
2381 match e {
2382 harn_lexer::LexerError::UnexpectedCharacter(_, span)
2383 | harn_lexer::LexerError::UnterminatedString(span)
2384 | harn_lexer::LexerError::UnterminatedBlockComment(span) => *span,
2385 }
2386}
2387
2388fn error_span_from_parse(e: &harn_parser::ParserError) -> harn_lexer::Span {
2389 match e {
2390 harn_parser::ParserError::Unexpected { span, .. } => *span,
2391 harn_parser::ParserError::UnexpectedEof { span, .. } => *span,
2392 }
2393}
2394
2395pub(crate) async fn execute(source: &str, source_path: Option<&Path>) -> Result<String, String> {
2397 execute_with_skill_dirs(source, source_path, &[]).await
2398}
2399
2400pub(crate) async fn execute_with_skill_dirs(
2401 source: &str,
2402 source_path: Option<&Path>,
2403 cli_skill_dirs: &[PathBuf],
2404) -> Result<String, String> {
2405 let mut lexer = Lexer::new(source);
2406 let tokens = lexer.tokenize().map_err(|e| e.to_string())?;
2407 let mut parser = Parser::new(tokens);
2408 let program = parser.parse().map_err(|e| e.to_string())?;
2409
2410 let mut checker = TypeChecker::new();
2415 if let Some(path) = source_path {
2416 let graph = harn_modules::build(&[path.to_path_buf()]);
2417 if let Some(imported) = graph.imported_names_for_file(path) {
2418 checker = checker.with_imported_names(imported);
2419 }
2420 if let Some(imported) = graph.imported_type_declarations_for_file(path) {
2421 checker = checker.with_imported_type_decls(imported);
2422 }
2423 if let Some(imported) = graph.imported_callable_declarations_for_file(path) {
2424 checker = checker.with_imported_callable_decls(imported);
2425 }
2426 }
2427 let type_diagnostics = checker.check(&program);
2428 let mut warning_lines = Vec::new();
2429 for diag in &type_diagnostics {
2430 match diag.severity {
2431 DiagnosticSeverity::Error => return Err(diag.message.clone()),
2432 DiagnosticSeverity::Warning => {
2433 warning_lines.push(format!("warning: {}", diag.message));
2434 }
2435 }
2436 }
2437
2438 let chunk = harn_vm::Compiler::new()
2439 .compile(&program)
2440 .map_err(|e| e.to_string())?;
2441
2442 let local = tokio::task::LocalSet::new();
2443 local
2444 .run_until(async {
2445 let mut vm = harn_vm::Vm::new();
2446 harn_vm::register_vm_stdlib(&mut vm);
2447 install_default_hostlib(&mut vm);
2448 let source_parent = source_path
2449 .and_then(|p| p.parent())
2450 .unwrap_or(std::path::Path::new("."));
2451 let project_root = harn_vm::stdlib::process::find_project_root(source_parent);
2452 let store_base = project_root.as_deref().unwrap_or(source_parent);
2453 let execution_cwd = std::env::current_dir()
2454 .unwrap_or_else(|_| std::path::PathBuf::from("."))
2455 .to_string_lossy()
2456 .into_owned();
2457 let source_dir = source_parent.to_string_lossy().into_owned();
2458 if source_path.is_some_and(is_conformance_path) {
2459 harn_vm::event_log::install_memory_for_current_thread(64);
2460 }
2461 harn_vm::register_store_builtins(&mut vm, store_base);
2462 harn_vm::register_metadata_builtins(&mut vm, store_base);
2463 let pipeline_name = source_path
2464 .and_then(|p| p.file_stem())
2465 .and_then(|s| s.to_str())
2466 .unwrap_or("default");
2467 harn_vm::register_checkpoint_builtins(&mut vm, store_base, pipeline_name);
2468 harn_vm::stdlib::process::set_thread_execution_context(Some(
2469 harn_vm::orchestration::RunExecutionRecord {
2470 cwd: Some(execution_cwd),
2471 source_dir: Some(source_dir),
2472 env: std::collections::BTreeMap::new(),
2473 adapter: None,
2474 repo_path: None,
2475 worktree_path: None,
2476 branch: None,
2477 base_ref: None,
2478 cleanup: None,
2479 },
2480 ));
2481 if let Some(ref root) = project_root {
2482 vm.set_project_root(root);
2483 }
2484 if let Some(path) = source_path {
2485 if let Some(parent) = path.parent() {
2486 if !parent.as_os_str().is_empty() {
2487 vm.set_source_dir(parent);
2488 }
2489 }
2490 }
2491 let loaded = skill_loader::load_skills(&skill_loader::SkillLoaderInputs {
2495 cli_dirs: cli_skill_dirs.to_vec(),
2496 source_path: source_path.map(Path::to_path_buf),
2497 });
2498 skill_loader::emit_loader_warnings(&loaded.loader_warnings);
2499 skill_loader::install_skills_global(&mut vm, &loaded);
2500 if let Some(path) = source_path {
2501 let extensions = package::load_runtime_extensions(path);
2502 package::install_runtime_extensions(&extensions);
2503 package::install_manifest_triggers(&mut vm, &extensions)
2504 .await
2505 .map_err(|error| format!("failed to install manifest triggers: {error}"))?;
2506 package::install_manifest_hooks(&mut vm, &extensions)
2507 .await
2508 .map_err(|error| format!("failed to install manifest hooks: {error}"))?;
2509 }
2510 let _event_log = harn_vm::event_log::active_event_log()
2511 .unwrap_or_else(|| harn_vm::event_log::install_memory_for_current_thread(64));
2512 let connector_clients_installed =
2513 should_install_default_connector_clients(source, source_path);
2514 if connector_clients_installed {
2515 install_default_connector_clients(store_base)
2516 .await
2517 .map_err(|error| format!("failed to initialize connector clients: {error}"))?;
2518 }
2519 let execution_result = vm.execute(&chunk).await.map_err(|e| e.to_string());
2520 harn_vm::egress::reset_egress_policy_for_host();
2521 if connector_clients_installed {
2522 harn_vm::clear_active_connector_clients();
2523 }
2524 harn_vm::stdlib::process::set_thread_execution_context(None);
2525 execution_result?;
2526 let mut output = String::new();
2527 for wl in &warning_lines {
2528 output.push_str(wl);
2529 output.push('\n');
2530 }
2531 output.push_str(vm.output());
2532 Ok(output)
2533 })
2534 .await
2535}
2536
2537fn should_install_default_connector_clients(source: &str, source_path: Option<&Path>) -> bool {
2538 if !source_path.is_some_and(is_conformance_path) {
2539 return true;
2540 }
2541 source.contains("connector_call")
2542 || source.contains("std/connectors")
2543 || source.contains("connectors/")
2544}
2545
2546fn is_conformance_path(path: &Path) -> bool {
2547 path.components()
2548 .any(|component| component.as_os_str() == "conformance")
2549}
2550
2551async fn install_default_connector_clients(base_dir: &Path) -> Result<(), String> {
2552 let event_log = harn_vm::event_log::active_event_log()
2553 .unwrap_or_else(|| harn_vm::event_log::install_memory_for_current_thread(64));
2554 let secret_namespace = connector_secret_namespace(base_dir);
2555 let secrets: Arc<dyn harn_vm::secrets::SecretProvider> = Arc::new(
2556 harn_vm::secrets::configured_default_chain(secret_namespace)
2557 .map_err(|error| format!("failed to configure secret providers: {error}"))?,
2558 );
2559
2560 let registry = harn_vm::ConnectorRegistry::default();
2561 let metrics = Arc::new(harn_vm::MetricsRegistry::default());
2562 let inbox = Arc::new(
2563 harn_vm::InboxIndex::new(event_log.clone(), metrics.clone())
2564 .await
2565 .map_err(|error| error.to_string())?,
2566 );
2567 registry
2568 .init_all(harn_vm::ConnectorCtx {
2569 event_log,
2570 secrets,
2571 inbox,
2572 metrics,
2573 rate_limiter: Arc::new(harn_vm::RateLimiterFactory::default()),
2574 })
2575 .await
2576 .map_err(|error| error.to_string())?;
2577 let clients = registry.client_map().await;
2578 harn_vm::install_active_connector_clients(clients);
2579 Ok(())
2580}
2581
2582fn connector_secret_namespace(base_dir: &Path) -> String {
2583 match std::env::var("HARN_SECRET_NAMESPACE") {
2584 Ok(namespace) if !namespace.trim().is_empty() => namespace,
2585 _ => {
2586 let leaf = base_dir
2587 .file_name()
2588 .and_then(|name| name.to_str())
2589 .filter(|name| !name.is_empty())
2590 .unwrap_or("workspace");
2591 format!("harn/{leaf}")
2592 }
2593 }
2594}
2595
2596#[cfg(test)]
2597mod main_tests {
2598 use super::{normalize_serve_args, should_install_default_connector_clients};
2599 use std::path::Path;
2600
2601 #[test]
2602 fn normalize_serve_args_inserts_a2a_for_legacy_shape() {
2603 let args = normalize_serve_args(vec![
2604 "harn".to_string(),
2605 "serve".to_string(),
2606 "--port".to_string(),
2607 "3000".to_string(),
2608 "agent.harn".to_string(),
2609 ]);
2610 assert_eq!(
2611 args,
2612 vec![
2613 "harn".to_string(),
2614 "serve".to_string(),
2615 "a2a".to_string(),
2616 "--port".to_string(),
2617 "3000".to_string(),
2618 "agent.harn".to_string(),
2619 ]
2620 );
2621 }
2622
2623 #[test]
2624 fn normalize_serve_args_preserves_explicit_subcommands() {
2625 let args = normalize_serve_args(vec![
2626 "harn".to_string(),
2627 "serve".to_string(),
2628 "acp".to_string(),
2629 "server.harn".to_string(),
2630 ]);
2631 assert_eq!(
2632 args,
2633 vec![
2634 "harn".to_string(),
2635 "serve".to_string(),
2636 "acp".to_string(),
2637 "server.harn".to_string(),
2638 ]
2639 );
2640 }
2641
2642 #[test]
2643 fn conformance_skips_connector_clients_unless_fixture_uses_connectors() {
2644 let path = Path::new("conformance/tests/language/basic.harn");
2645 assert!(!should_install_default_connector_clients(
2646 "println(1)",
2647 Some(path)
2648 ));
2649 assert!(!should_install_default_connector_clients(
2650 "trust_graph_verify_chain()",
2651 Some(path)
2652 ));
2653 assert!(should_install_default_connector_clients(
2654 "import { post_message } from \"std/connectors/slack\"",
2655 Some(path)
2656 ));
2657 assert!(should_install_default_connector_clients(
2658 "println(1)",
2659 Some(Path::new("examples/demo.harn"))
2660 ));
2661 }
2662}