llman 0.0.30

A tool for managing LLM application rules(prompts) ...
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

use crate::x::sdd_eval::{paths, playbook, run};
use anyhow::{Context, Result};
use clap::{Args, Subcommand};
use std::path::PathBuf;

#[derive(Args)]
#[command(
    author,
    version,
    about,
    long_about = "Playbook-driven SDD evaluation pipeline (experimental)",
    subcommand_required = true,
    arg_required_else_help = true
)]
pub struct SddEvalArgs {
    #[command(subcommand)]
    pub command: SddEvalCommand,
}

#[derive(Subcommand)]
pub enum SddEvalCommand {
    /// Create a playbook template under .llman/sdd-eval/playbooks/
    Init {
        /// Playbook name (without extension)
        #[arg(long)]
        name: String,
        /// Overwrite existing playbook file
        #[arg(long)]
        force: bool,
    },
    /// Run a playbook and create a new run directory
    Run {
        /// Path to the playbook YAML file
        #[arg(long)]
        playbook: PathBuf,
    },
    /// Generate (or re-generate) a report for a run
    Report {
        /// Run id (directory name under .llman/sdd-eval/runs/)
        #[arg(long)]
        run: String,
    },
    /// Import human scores and merge into run report data
    #[command(name = "import-human")]
    ImportHuman {
        /// Run id (directory name under .llman/sdd-eval/runs/)
        #[arg(long)]
        run: String,
        /// JSON file containing human scores
        #[arg(long)]
        file: PathBuf,
    },
}

pub fn run(args: &SddEvalArgs) -> Result<()> {
    let project_root = paths::project_root_from_cwd()?;

    match &args.command {
        SddEvalCommand::Init { name, force } => {
            let path = paths::playbook_path(&project_root, name);
            playbook::write_template(&path, *force)
                .with_context(|| format!("write playbook template {}", path.display()))?;
            println!("{}", path.display());
            Ok(())
        }
        SddEvalCommand::Run { playbook } => {
            let pb = playbook::load_from_path(playbook)?;
            pb.validate().context("validate playbook")?;
            let run_dir = run::create_run(&project_root, playbook, &pb)?;
            run::execute_run(&project_root, &run_dir, &pb)?;
            println!("{}", run_dir.display());
            Ok(())
        }
        SddEvalCommand::Report { run: run_id } => {
            run::generate_report(&project_root, run_id)?;
            Ok(())
        }
        SddEvalCommand::ImportHuman { run: run_id, file } => {
            run::import_human_scores(&project_root, run_id, file)?;
            Ok(())
        }
    }
}